Merge pull request #581 from NREL/pso_update

update PSO

.travis.yml

@@ -47,11 +47,11 @@ jobs:
       env: BUILD_TYPE=integration
     - stage: test
       os: osx
-      osx_image: xcode9.2
+      osx_image: xcode9.4
       env: BUILD_TYPE=test
     - stage: test
       os: osx
-      osx_image: xcode9.2
+      osx_image: xcode9.4
       env: BUILD_TYPE=integration
     - stage: test
       os: linux
@@ -79,7 +79,7 @@ jobs:
         - git checkout -- .dockerignore && git checkout -- Dockerfile
     # - stage: Upload native OSX build
     #   if: branch=local-builds
-    #   os: xcode9.2
+    #   os: xcode9.4
     #   sudo: required
     #   env:
     #     - BUILD_TYPE=publish

Dockerfile

@@ -102,8 +102,6 @@ ADD .rubocop.yml /opt/openstudio/.rubocop.yml
 RUN rm Gemfile.lock
 RUN bundle install --jobs=3 --retry=3
 
-# Configure IPVS keepalive
-ADD /docker/server/ipvs-keepalive.conf /etc/sysctl.d/ipvs-keepalive.conf
 RUN sudo sysctl --system
 
 # Add in scripts for running server. This includes the wait-for-it scripts to ensure other processes (mongo, redis) have
@@ -114,16 +112,12 @@ COPY /docker/server/run-server-tests.sh /usr/local/bin/run-server-tests
 COPY /docker/server/rails-entrypoint.sh /usr/local/bin/rails-entrypoint
 COPY /docker/server/start-web-background.sh /usr/local/bin/start-web-background
 COPY /docker/server/start-workers.sh /usr/local/bin/start-workers
-COPY /docker/server/memfix-controller.rb /usr/local/lib/memfix-controller.rb
-COPY /docker/server/memfix.rb /usr/local/lib/memfix.rb
 RUN chmod 755 /usr/local/bin/wait-for-it
 RUN chmod +x /usr/local/bin/start-server
 RUN chmod +x /usr/local/bin/run-server-tests
 RUN chmod 755 /usr/local/bin/rails-entrypoint
 RUN chmod 755 /usr/local/bin/start-web-background
 RUN chmod 755 /usr/local/bin/start-workers
-RUN chmod +x /usr/local/lib/memfix-controller.rb
-RUN chmod +x /usr/local/lib/memfix.rb
 
 # set the permissions for windows users
 RUN chmod +x /opt/openstudio/server/bin/*

docker-compose.deploy.yml

@@ -49,6 +49,7 @@ services:
       - queue
       - web
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager
@@ -69,6 +70,7 @@ services:
       - db
       - rserve
     deploy:
+      endpoint_mode: dnsrr
       resources:
         reservations:
           cpus: "1"
@@ -82,6 +84,7 @@ services:
       - web-background
       - db
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager

docker/R/Dockerfile

@@ -2,7 +2,7 @@
 # DESCRIPTION:      OpenStudio Server R Container
 # TO_BUILD_AND_RUN: docker-compose up
 
-FROM nrel/openstudio-r:3.6.1
+FROM nrel/openstudio-r:3.6.1-1
 MAINTAINER Nicholas Long nicholas.long@nrel.gov
 
 # Add in the additional R packages
@@ -17,17 +17,13 @@ RUN R CMD INSTALL /opt/openstudio/R/lib/R-packages/NRELmoo*.tar.gz /opt/openstud
 ADD /lib /opt/openstudio/R/lib
 
 # Install Gems
-RUN gem install bundler -v1.16.4
+RUN gem install bundler -v2.1.0
 RUN cd /opt/openstudio/R/lib && bundle install --jobs=3 --retry=3
 
 ADD /start.R /start.R
 ADD /Rserv.conf /etc/Rserv.conf
 RUN mkdir -p /mnt/openstudio/log && chmod 775 /mnt/openstudio/log
 
-# Configure keepalive for IPVS
-ADD /ipvs-keepalive.conf /etc/sysctl.d/ipvs-keepalive.conf
-RUN sudo sysctl --system
-
 ADD /rserve-entrypoint.sh /usr/local/bin/rserve-entrypoint
 RUN chmod 755 /usr/local/bin/rserve-entrypoint
 ENTRYPOINT ["rserve-entrypoint"]

docker/R/Rserv.conf

@@ -2,4 +2,4 @@ remote enable
 keep.alive enable
 auth disable
 fileio enable
-maxinbuf 500000
+maxinbuf 0

docker/R/lib/Gemfile

@@ -1,3 +1,3 @@
 source 'http://rubygems.org'
 
-gem 'rest-client', '1.8.0'
+gem 'rest-client', '~>2.0.1' 

docker/R/lib/R-packages-code/NRELpso/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: NRELpso
 Type: Package
 Title: Particle Swarm Optimisation, with focus on Environmental Models
-Version: 0.3.13
-Date: 2014-04-11
+Version: 0.5-1
+Date: 2020-04-29
 Author: Mauricio Zambrano-Bigiarini [aut, cre], Rodrigo Rojas [ctb]
 Authors@R: c(person("Mauricio", "Zambrano-Bigiarini", email="mzb.devel@gmail.com", role=c("aut","cre")), person("Rodrigo", "Rojas", email="Rodrigo.RojasMujica@gmail.com", role=c("ctb")) )
 Maintainer: Mauricio Zambrano-Bigiarini <mzb.devel@gmail.com>

docker/R/lib/R-packages-code/NRELpso/R/PSO_v2013.R

@@ -1,6 +1,8 @@
-# File PSO_v2012.R
-# Part of the hydroPSO package, http://www.rforge.net/hydroPSO/
-# Copyright 2008-2014 Mauricio Zambrano-Bigiarini
+# File PSO_v2013.R
+# Part of the hydroPSO R package, https://github.com/hzambran/hydroPSO
+#                                 http://cran.r-project.org/web/packages/hydroPSO
+#                                 http://www.rforge.net/hydroPSO/
+# Copyright 2010-2020 Mauricio Zambrano-Bigiarini
 # Distributed under GPL 2 or later
 
 ################################################################################
@@ -216,6 +218,7 @@ compute.CF <- function(c1, c2) {
 # Created: 2008                                                                #
 # Updates: Oct-2011 ; Nov-2011                                                 #
 #          19-Sep-2012 ; 20-Sep-2012 ; 28-Oct-2012 ; 31-Oct-2012               #
+#          09-May-2016                                                         #
 ################################################################################
 compute.veloc <- function(x, v, w, c1, c2, CF, Pbest, part.index, gbest, 
                           topology, method, MinMax, neighs.index, 
@@ -283,7 +286,7 @@ compute.veloc <- function(x, v, w, c1, c2, CF, Pbest, part.index, gbest,
 
 
                    } else if ( method=="wfips" ) {
-
+                       
                        neighs.index <- neighs.index[!is.na(neighs.index)] # only for topology=='random' 
                        N    <- length(neighs.index)              
                        X    <- matrix(rep(x,N), nrow=N, byrow=TRUE)
@@ -292,8 +295,8 @@ compute.veloc <- function(x, v, w, c1, c2, CF, Pbest, part.index, gbest,
                        phi  <- c1 + c2
                        r    <- runif(N, min=0, max=phi)
                        if(MinMax == "min") {
-                         wght <- (1/lpbest.fit)/sum(1/lpbest.fit)
-                       } else wght <- lpbest.fit/sum(lpbest.fit) 
+                         wght <- (1/pfit)/sum(1/pfit)
+                       } else wght <- pfit/sum(pfit) 
 
                        vn  <-  CF * ( w*v + (1/N) * colSums( wght*r*(P-X) ) )	
 
@@ -1282,6 +1285,10 @@ Random.Topology.Generation <- function(npart, K,
 #          19-Dec-2012                                                         #
 #          07-May-2013 ; 10-May-2013 ; 28-May-2013 ; 29-May-2013               #
 #          07-Feb-2014 ; 09-Abr-2014                                           #
+#          29-Jan-2016 ; 09-May-2016                                           #
+#          10-Jun-2018                                                         #
+#          27-Feb-2020 ; 28-Feb-2020 ; 06-Mar-2020 ; 09-Mar-2020 ; 12-Mar-2020 #
+#          13-Mar-2020 ; 24-Apr-2020                                           #
 ################################################################################
 # 'lower'           : minimum possible value for each parameter
 # 'upper'           : maximum possible value for each parameter
@@ -1692,7 +1699,7 @@ NRELpso <- function(cl=cl,
 
     ############################################################################  
     # 1)                              Initialisation                           #
-    ###################################################$$$$#####################  
+    ############################################################################  
     if (verbose) message("                                                                                ")          
     if (verbose) message("================================================================================")
     if (verbose) message("[                                Initialising  ...                             ]")
@@ -1756,14 +1763,14 @@ NRELpso <- function(cl=cl,
       } else abstol <- -Inf
 
     if (Xini.type=="lhs") { 
-	if ( is.na( match("lhs", installed.packages()[,"Package"] ) ) ) {
+	if ( length(find.package("lhs", quiet=TRUE)) == 0 ) {
 	    warning("[ Package 'lhs' is not installed =>  Xini.type='random' ]")
 	    Xini.type <- "random"
 	}  # IF end  
     } # IF end
 
     if (Vini.type %in% c("lhs2011", "lhs2007")) { 
-	if ( is.na( match("lhs", installed.packages()[,"Package"] ) ) ) {
+	if ( length(find.package("lhs", quiet=TRUE)) == 0 ) {
 	    warning("[ Package 'lhs' is not installed =>  Vini.type='random2011' ]")
 	    Vini.type <- "random2011"
 	}  # IF end  
@@ -1874,10 +1881,10 @@ NRELpso <- function(cl=cl,
     X <- InitializateX(npart=npart, x.MinMax=X.Boundaries, x.ini.type=Xini.type)
     V <- InitializateV(npart=npart, x.MinMax=X.Boundaries, v.ini.type=Vini.type, 
                        Xini=X)
-    V <- t(apply(V, MARGIN=1, FUN=velocity.boundary.treatment, vmax=Vmax))
+    V <- matrix( apply(V, MARGIN=1, FUN=velocity.boundary.treatment, vmax=Vmax), nrow=npart,byrow=TRUE)
 
     if (!missing(par)) {
-      if (!any(is.na(par)) && all(par>=lower) && all(par<=upper)) { 
+      if (!any(is.na(par)) && all(t(par)>=lower) && all(t(par)<=upper)) { 
 	if (class(par)=="numeric") {
 	  X[1,] <- par 
 	} else if ( (class(par)=="matrix") | (class(par)=="data.frame") ) {
@@ -2444,7 +2451,7 @@ NRELpso <- function(cl=cl,
 				localBest.pos=LocalBest.pos[j],            # topology=c("random", "lbest")
 				ngbest.fit=ngbest.fit,                     # topology="ipso"
 				ngbest=X.best.part[ngbest.pos, ],          # topology="ipso"
-				lpbest.fit= pbest.fit[X.neighbours[j, ]]   # method="wfips"
+				lpbest.fit= pbest.fit                      # method="wfips"
 				)  
 
 	V[j,] <- velocity.boundary.treatment(v= V[j,], vmax=Vmax)

docker/R/lib/R-packages-code/NRELpso/R/test_functions.R

@@ -1,5 +1,5 @@
 # Part of the hydroPSO package, http://www.rforge.net/hydroPSO/
-# Copyright 2008-2012 Mauricio Zambrano-Bigiarini & Rodrigo Rojas
+# Copyright 2008-2018 Mauricio Zambrano-Bigiarini & Rodrigo Rojas
 # Distributed under GPL 2 or later
 
 # All these function were started on 2008, with updates on:                    #
@@ -15,31 +15,31 @@ sinc <- function(x) {
 } # 'sinc' END
 
 
-# MZB, RR, 21-Jun-2011,  14-Nov-2011 ; 21-Nov-2012
+# MZB, RR, 21-Jun-2011,  14-Nov-2011 ; 21-Nov-2012 ; 10-Jun-2018
 # Rosenbrock function: f(1,..,1)=0. Minimization. In [-30, 30]^n. AcceptableError < 100
 # Properties : Unimodal, Non-separable 
 # Description: The Rosenbrock function is non-convex, unimodal and non-separable. 
 #              It is also known as \emph{Rosenbrock's valley} or \emph{Rosenbrock's banana} function.
 #              The global minimum is inside a long, narrow, parabolic shaped flat valley. 
 #              To find the valley is trivial. To converge to the global minimum, however, is difficult. 
 #              It only has one optimum located at the point \preformatted{o =(1,...,1)}. 
-#              It is a quadratic function, and its search range is [−30, 30] for each variable. 
+#              It is a quadratic function, and its search range is [-30, 30] for each variable. 
 # Ref: http://en.wikipedia.org/wiki/Rosenbrock_function, http://www.it.lut.fi/ip/evo/functions/node5.html
 rosenbrock <- function(x) {  
   n <- length(x)
   return( sum( 100*( x[2:n] - x[1:(n-1)]^2 )^2 + ( x[1:(n-1)] - 1 )^2 ) )
 } # 'rosenbrock' END
 
 
-# MZB, RR, 21-Jun-2011; 21-Nov-2012
+# MZB, RR, 21-Jun-2011; 21-Nov-2012 ; 10-Jun-2018
 # Sphere function: f(0,..,0)=0. Minimization. In [-100, 100]^n. AcceptableError < 0.01
 # Properties : Unimodal, additively separable
 # Description: The Sphere test function is one of the most simple test functions 
 #              available in the specialized literature. This unimodal and separable 
 #              test function can be scaled up to any number of variables. 
 #              It belongs to a family of functions called quadratic functions and 
 #              only has one optimum in the point o = (0,...,0). The search range 
-#              commonly used for the Sphere function is [−100, 100] for each decision variable.
+#              commonly used for the Sphere function is [-100, 100] for each decision variable.
 # Reference  : http://www.it.lut.fi/ip/evo/functions/node2.html
 sphere <- function(x) {
   return(sum(x^2))  

docker/R/lib/api_create_datapoint.rb

@@ -38,6 +38,7 @@
 
 require 'optparse'
 require 'rest-client'
+require 'json'
 
 options = { submit_simulation: false, sleep_time: 5 }
 o = OptionParser.new do |opts|
@@ -73,7 +74,7 @@
     }
   }
 
-  a = RestClient.post "#{options[:host]}/analyses/#{options[:analysis_id]}/data_points.json", data_point_data
+  a = RestClient.post "#{options[:host]}/analyses/#{options[:analysis_id]}/data_points.json", data_point_data.to_json , {content_type: :json, accept: :json}
   raise 'Could not create datapoint' unless a.code == 201
 
   a = JSON.parse(a, symbolize_names: true)

docker/R/lib/api_get_status.rb

@@ -59,7 +59,7 @@
 result = {}
 result[:status] = false
 begin
-  a = RestClient.get "#{options[:host]}/analyses/#{options[:analysis_id]}/status.json"
+  a = RestClient.get "#{options[:host]}/analyses/#{options[:analysis_id]}/status.json" , {accept: :json}
   # TODO: retries?
   raise 'Could not create datapoint' unless a.code == 200
 

docker/R/lib/pso.R

@@ -141,10 +141,11 @@ if (topology == "vonneumann") {topology <- "vonNeumann"}
 print(paste("topology:", topology))
 print(paste("c1:", c1))
 print(paste("c2:", c2))
-print(paste("lambda:", lambda))
+print(paste("lambda:", lambda))
+print(paste("report:", report))
 
 results <- NULL
-try(results <- NRELpso(cl=cl, fn=f, lower=varMin, upper=varMax, method=method, control=list(write2disk=FALSE, parallel="true", npart=npart, maxit=maxit, maxfn=maxfn, abstol=abstol, reltol=reltol, Xini.type=xini, Vini.type=vini, boundary.wall=boundary, topology=topology, c1=c1, c2=c2, lambda=lambda)), silent=FALSE)
+try(results <- NRELpso(cl=cl, fn=f, lower=varMin, upper=varMax, method=method, control=list(write2disk=FALSE, parallel="true", npart=npart, maxit=maxit, maxfn=maxfn, abstol=abstol, reltol=reltol, Xini.type=xini, Vini.type=vini, boundary.wall=boundary, topology=topology, c1=c1, c2=c2, lambda=lambda, REPORT=report)), silent=FALSE)
 #print(paste("scp command:",scp))
 #print(paste("scp command:",scp2))
 #system(scp,intern=TRUE)
@@ -186,4 +187,7 @@ if (!file.exists(bestresults_filename) && !is.null(results$par)) {
   write.table(answer, file=bestresults_filename, quote=FALSE,row.names=FALSE,col.names=FALSE)
   convergenceflag <- paste('{',paste('"',"exit_on_guideline_14",'"',': ',"false",sep='', collapse=','),'}',sep='')
   write(convergenceflag, file=paste(analysis_dir,"/convergence_flag.json",sep=''))
-}
+}
+print("stopping cluster in pso.R")
+stopCluster(cl)
+print("cluster stopped")

docker/deployment/docker-compose.aws.yml

@@ -60,6 +60,7 @@ services:
       - web
       - queue
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager
@@ -81,6 +82,7 @@ services:
       - queue
       - rserve
     deploy:
+      endpoint_mode: dnsrr
       resources:
         reservations:
           cpus: "1"
@@ -94,6 +96,7 @@ services:
       - web-background
       - db
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager

local_setup_scripts/docker-compose.yml

@@ -60,6 +60,7 @@ services:
       - web
       - queue
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager
@@ -81,6 +82,7 @@ services:
       - queue
       - rserve
     deploy:
+      endpoint_mode: dnsrr
       resources:
         reservations:
           cpus: "1"
@@ -94,6 +96,7 @@ services:
       - web-background
       - db
     deploy:
+      endpoint_mode: dnsrr
       placement:
         constraints:
           - node.role == manager

local_setup_scripts/rebuild_sr.sh

@@ -5,9 +5,9 @@ docker stack rm osserver || true
 docker volume rm osdata -f || true
 docker volume rm dbdata -f || true
 
-while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
+#while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
 #docker image rm 127.0.0.1:5000/openstudio-server -f
-docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.0.1-rc1
+docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.0.1
 docker push 127.0.0.1:5000/openstudio-server
 cd docker/R/
 #docker image rm 127.0.0.1:5000/openstudio-rserve -f
@@ -23,6 +23,6 @@ docker push 127.0.0.1:5000/redis
 cd ../../local_setup_scripts
 docker stack deploy osserver --compose-file=./docker-compose.yml
 while ( nc -zv 127.0.0.1 80 3>&1 1>&2- 2>&3- ) | awk -F ":" '$3 != " Connection refused" {exit 1}'; do sleep 5; done
-docker service scale osserver_worker=1
+docker service scale osserver_worker=42
 echo 'osserver stack rebuilt and redeployed'