From ffa4e20c2fd1f1c0de06aba6bcf3022a45f6728f Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 13:53:44 -0500 Subject: [PATCH 01/21] New Query General module, used by both Restbroker and EC2-Query --- .idea-modules/query-general/query-general.iml | 14 +++ query/build.properties | 12 ++ query/build.xml | 116 ++++++++++++++++++ .../security/FileUserDetailsService.java | 19 +-- .../querygeneral}/security/QueryUser.java | 3 +- .../security/QueryUserDetailsService.java | 3 +- 6 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 .idea-modules/query-general/query-general.iml create mode 100644 query/build.properties create mode 100644 query/build.xml rename {messaging/query/java/source/src/org/nimbustools/messaging/query => query/src/org/nimbustools/querygeneral}/security/FileUserDetailsService.java (87%) rename {messaging/query/java/source/src/org/nimbustools/messaging/query => query/src/org/nimbustools/querygeneral}/security/QueryUser.java (98%) rename {messaging/query/java/source/src/org/nimbustools/messaging/query => query/src/org/nimbustools/querygeneral}/security/QueryUserDetailsService.java (95%) diff --git a/.idea-modules/query-general/query-general.iml b/.idea-modules/query-general/query-general.iml new file mode 100644 index 00000000..520b6ab7 --- /dev/null +++ b/.idea-modules/query-general/query-general.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/query/build.properties b/query/build.properties new file mode 100644 index 00000000..bf5549ab --- /dev/null +++ b/query/build.properties @@ -0,0 +1,12 @@ +nimbus.query.main.print-noun=Query-General Module + +nimbus.query.main.build.dir=build +nimbus.query.main.build.dest=build/classes +nimbus.query.main.src.dir=src +nimbus.query.main.dist.dir=dist +nimbus.query.main.jar.name=nimbus-query.jar + +nimbus.query.main.lib.dir=../lib/services + +# you can override this in ~/nimbus.build.properties +nimbus.java.compilerarg=-nowarn diff --git a/query/build.xml b/query/build.xml new file mode 100644 index 00000000..88d1ec9b --- /dev/null +++ b/query/build.xml @@ -0,0 +1,116 @@ + + + + + Query-General Module + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Building: ${nimbus.query.main.print-noun} + + Built: ${nimbus.query.main.print-noun} + + + + + + + + + + + + + + diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/FileUserDetailsService.java b/query/src/org/nimbustools/querygeneral/security/FileUserDetailsService.java similarity index 87% rename from messaging/query/java/source/src/org/nimbustools/messaging/query/security/FileUserDetailsService.java rename to query/src/org/nimbustools/querygeneral/security/FileUserDetailsService.java index ab4ba50b..0047c48a 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/FileUserDetailsService.java +++ b/query/src/org/nimbustools/querygeneral/security/FileUserDetailsService.java @@ -1,19 +1,4 @@ -/* - * Copyright 1999-2009 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy - * of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.nimbustools.messaging.query.security; +package org.nimbustools.querygeneral.security; import org.springframework.core.io.Resource; import org.springframework.security.core.userdetails.UsernameNotFoundException; @@ -34,7 +19,7 @@ import java.util.NoSuchElementException; /** - * Placeholder UserDetailsService which loads users from a file. + * UserDetailsService which loads users from a file. * * The file format is one user per line: DN accessID secret * Any token may be quoted and they are separated by whitespace. diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUser.java b/query/src/org/nimbustools/querygeneral/security/QueryUser.java similarity index 98% rename from messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUser.java rename to query/src/org/nimbustools/querygeneral/security/QueryUser.java index 34f8b175..040cb78d 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUser.java +++ b/query/src/org/nimbustools/querygeneral/security/QueryUser.java @@ -13,7 +13,8 @@ * License for the specific language governing permissions and limitations * under the License. */ -package org.nimbustools.messaging.query.security; + +package org.nimbustools.querygeneral.security; import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.core.GrantedAuthority; diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUserDetailsService.java b/query/src/org/nimbustools/querygeneral/security/QueryUserDetailsService.java similarity index 95% rename from messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUserDetailsService.java rename to query/src/org/nimbustools/querygeneral/security/QueryUserDetailsService.java index ab079a67..2623bdfc 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryUserDetailsService.java +++ b/query/src/org/nimbustools/querygeneral/security/QueryUserDetailsService.java @@ -13,7 +13,8 @@ * License for the specific language governing permissions and limitations * under the License. */ -package org.nimbustools.messaging.query.security; + +package org.nimbustools.querygeneral.security; import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.core.userdetails.UsernameNotFoundException; From ade5d7b10e5ee9beea6d2667061569e8b3926ae3 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 13:56:44 -0500 Subject: [PATCH 02/21] The usages of new Query General module --- .../src/org/nimbustools/ctxbroker/rest/BrokerResource.java | 2 +- messaging/query/java/source/build.properties | 1 + messaging/query/java/source/build.xml | 5 +++-- .../query/security/NimbusAuthzUserDetailsService.java | 2 ++ .../messaging/query/security/QueryAuthenticationFilter.java | 2 ++ .../messaging/query/security/QueryAuthenticationToken.java | 1 + .../messaging/query/security/QueryContainerInterface.java | 2 ++ 7 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ctx-broker/src/org/nimbustools/ctxbroker/rest/BrokerResource.java b/ctx-broker/src/org/nimbustools/ctxbroker/rest/BrokerResource.java index f8f5d329..a8731151 100644 --- a/ctx-broker/src/org/nimbustools/ctxbroker/rest/BrokerResource.java +++ b/ctx-broker/src/org/nimbustools/ctxbroker/rest/BrokerResource.java @@ -30,7 +30,7 @@ import org.nimbustools.ctxbroker.service.ContextBrokerHomeImpl; import org.nimbustools.ctxbroker.service.ContextBrokerResourceImpl; import org.nimbustools.ctxbroker.service.ContextBrokerServiceImpl; -import org.nimbustools.messaging.query.security.QueryUser; +import org.nimbustools.querygeneral.security.QueryUser; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; diff --git a/messaging/query/java/source/build.properties b/messaging/query/java/source/build.properties index 9d0c1132..cfb36e1c 100644 --- a/messaging/query/java/source/build.properties +++ b/messaging/query/java/source/build.properties @@ -13,6 +13,7 @@ nimbus.messaging.gt4_0.common.dist.dir=../../../gt4.0/java/common/dist/ nimbus.messaging.gt4_0-elastic.dist.dir=../../../gt4.0-elastic/java/msgbridge/dist/ nimbus.service.api.dist.dir=../../../../service-api/java/source/dist/ nimbus.authzdb.dist.dir=../../../../authzdb/dist/ +nimbus.querygen.dist.dir=../../../../query/dist nimbus.service.dist.dir=../../../../service/service/java/source/dist/ # (gar is the 'dist' of stubs but we only want the jars... will be better later) diff --git a/messaging/query/java/source/build.xml b/messaging/query/java/source/build.xml index e55bb892..2c5bfc2f 100644 --- a/messaging/query/java/source/build.xml +++ b/messaging/query/java/source/build.xml @@ -74,13 +74,14 @@ - + - + + diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/NimbusAuthzUserDetailsService.java b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/NimbusAuthzUserDetailsService.java index f24213bd..071d7263 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/NimbusAuthzUserDetailsService.java +++ b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/NimbusAuthzUserDetailsService.java @@ -20,6 +20,8 @@ import org.nimbus.authz.AuthzDBAdapter; import org.nimbus.authz.AuthzDBException; import org.nimbus.authz.UserAlias; +import org.nimbustools.querygeneral.security.QueryUser; +import org.nimbustools.querygeneral.security.QueryUserDetailsService; import org.springframework.beans.factory.InitializingBean; import org.springframework.dao.DataAccessException; import org.springframework.security.core.userdetails.UsernameNotFoundException; diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationFilter.java b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationFilter.java index 7de603d8..a9a052b8 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationFilter.java +++ b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationFilter.java @@ -18,6 +18,8 @@ import org.apache.commons.codec.binary.Base64; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.nimbustools.querygeneral.security.QueryUser; +import org.nimbustools.querygeneral.security.QueryUserDetailsService; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.userdetails.UsernameNotFoundException; import org.springframework.web.filter.GenericFilterBean; diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationToken.java b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationToken.java index 1bb2e680..8ca87514 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationToken.java +++ b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryAuthenticationToken.java @@ -15,6 +15,7 @@ */ package org.nimbustools.messaging.query.security; +import org.nimbustools.querygeneral.security.QueryUser; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.Authentication; diff --git a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryContainerInterface.java b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryContainerInterface.java index f9c9db69..a9ce52c5 100644 --- a/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryContainerInterface.java +++ b/messaging/query/java/source/src/org/nimbustools/messaging/query/security/QueryContainerInterface.java @@ -15,6 +15,8 @@ */ package org.nimbustools.messaging.query.security; +import org.nimbustools.querygeneral.security.QueryUser; +import org.nimbustools.querygeneral.security.QueryUserDetailsService; import org.nimbustools.api._repr._Caller; import org.nimbustools.api.brain.ModuleLocator; import org.nimbustools.api.repr.Caller; From 5550a881f3195e95b363374ecd5bafbc4d2d2bae Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 14:09:47 -0500 Subject: [PATCH 03/21] lib/workspaceservice is for jars in lib/services that are only used by IaaS --- .idea-modules/ctx/contextualization.iml | 9 ++- .idea-modules/query/ec2query.iml | 3 +- .idea-modules/service-suites/tests-common.iml | 2 +- .idea-modules/wrksp-svc/workspace-service.iml | 3 +- .idea/libraries/ctxbroker_libs.xml | 15 ----- .idea/libraries/workspace_service_libs.xml | 29 --------- .idea/modules.xml | 1 + .../commons-dbcp.jar | Bin .../commons-io-1.4.jar | Bin .../commons-pool.jar | Bin lib/{services => workspaceservice}/derby.jar | Bin .../derbyclient.jar | Bin .../derbynet.jar | Bin .../derbyrun.jar | Bin .../derbytools.jar | Bin .../junixsocket-1.3.jar | Bin .../junixsocket-rmi-1.3.jar | Bin .../sqlitejdbc-v056.jar | Bin .../gt4.0/java/gar-builder/build.properties | 8 ++- messaging/gt4.0/java/gar-builder/build.xml | 56 +++++++++++++----- metadata/java/source/build.properties | 1 + metadata/java/source/build.xml | 4 ++ service/service/java/source/build.properties | 3 +- service/service/java/source/build.xml | 3 + 24 files changed, 66 insertions(+), 71 deletions(-) delete mode 100644 .idea/libraries/ctxbroker_libs.xml delete mode 100644 .idea/libraries/workspace_service_libs.xml rename lib/{services => workspaceservice}/commons-dbcp.jar (100%) rename lib/{services => workspaceservice}/commons-io-1.4.jar (100%) rename lib/{services => workspaceservice}/commons-pool.jar (100%) rename lib/{services => workspaceservice}/derby.jar (100%) rename lib/{services => workspaceservice}/derbyclient.jar (100%) rename lib/{services => workspaceservice}/derbynet.jar (100%) rename lib/{services => workspaceservice}/derbyrun.jar (100%) rename lib/{services => workspaceservice}/derbytools.jar (100%) rename lib/{services => workspaceservice}/junixsocket-1.3.jar (100%) rename lib/{services => workspaceservice}/junixsocket-rmi-1.3.jar (100%) rename lib/{services => workspaceservice}/sqlitejdbc-v056.jar (100%) diff --git a/.idea-modules/ctx/contextualization.iml b/.idea-modules/ctx/contextualization.iml index 3fe06d21..7f0ee884 100644 --- a/.idea-modules/ctx/contextualization.iml +++ b/.idea-modules/ctx/contextualization.iml @@ -9,13 +9,12 @@ - - - - - + + + + diff --git a/.idea-modules/query/ec2query.iml b/.idea-modules/query/ec2query.iml index f147d72c..6453abc0 100644 --- a/.idea-modules/query/ec2query.iml +++ b/.idea-modules/query/ec2query.iml @@ -22,8 +22,9 @@ - + + diff --git a/.idea-modules/service-suites/tests-common.iml b/.idea-modules/service-suites/tests-common.iml index 385aa16f..42826f29 100644 --- a/.idea-modules/service-suites/tests-common.iml +++ b/.idea-modules/service-suites/tests-common.iml @@ -10,7 +10,7 @@ - + diff --git a/.idea-modules/wrksp-svc/workspace-service.iml b/.idea-modules/wrksp-svc/workspace-service.iml index e68d1917..d9d2c4e8 100644 --- a/.idea-modules/wrksp-svc/workspace-service.iml +++ b/.idea-modules/wrksp-svc/workspace-service.iml @@ -30,10 +30,11 @@ - + + diff --git a/.idea/libraries/ctxbroker_libs.xml b/.idea/libraries/ctxbroker_libs.xml deleted file mode 100644 index c2d05ffd..00000000 --- a/.idea/libraries/ctxbroker_libs.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/workspace_service_libs.xml b/.idea/libraries/workspace_service_libs.xml deleted file mode 100644 index 1e34236f..00000000 --- a/.idea/libraries/workspace_service_libs.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 3f1253d4..49ce351f 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -12,6 +12,7 @@ + diff --git a/lib/services/commons-dbcp.jar b/lib/workspaceservice/commons-dbcp.jar similarity index 100% rename from lib/services/commons-dbcp.jar rename to lib/workspaceservice/commons-dbcp.jar diff --git a/lib/services/commons-io-1.4.jar b/lib/workspaceservice/commons-io-1.4.jar similarity index 100% rename from lib/services/commons-io-1.4.jar rename to lib/workspaceservice/commons-io-1.4.jar diff --git a/lib/services/commons-pool.jar b/lib/workspaceservice/commons-pool.jar similarity index 100% rename from lib/services/commons-pool.jar rename to lib/workspaceservice/commons-pool.jar diff --git a/lib/services/derby.jar b/lib/workspaceservice/derby.jar similarity index 100% rename from lib/services/derby.jar rename to lib/workspaceservice/derby.jar diff --git a/lib/services/derbyclient.jar b/lib/workspaceservice/derbyclient.jar similarity index 100% rename from lib/services/derbyclient.jar rename to lib/workspaceservice/derbyclient.jar diff --git a/lib/services/derbynet.jar b/lib/workspaceservice/derbynet.jar similarity index 100% rename from lib/services/derbynet.jar rename to lib/workspaceservice/derbynet.jar diff --git a/lib/services/derbyrun.jar b/lib/workspaceservice/derbyrun.jar similarity index 100% rename from lib/services/derbyrun.jar rename to lib/workspaceservice/derbyrun.jar diff --git a/lib/services/derbytools.jar b/lib/workspaceservice/derbytools.jar similarity index 100% rename from lib/services/derbytools.jar rename to lib/workspaceservice/derbytools.jar diff --git a/lib/services/junixsocket-1.3.jar b/lib/workspaceservice/junixsocket-1.3.jar similarity index 100% rename from lib/services/junixsocket-1.3.jar rename to lib/workspaceservice/junixsocket-1.3.jar diff --git a/lib/services/junixsocket-rmi-1.3.jar b/lib/workspaceservice/junixsocket-rmi-1.3.jar similarity index 100% rename from lib/services/junixsocket-rmi-1.3.jar rename to lib/workspaceservice/junixsocket-rmi-1.3.jar diff --git a/lib/services/sqlitejdbc-v056.jar b/lib/workspaceservice/sqlitejdbc-v056.jar similarity index 100% rename from lib/services/sqlitejdbc-v056.jar rename to lib/workspaceservice/sqlitejdbc-v056.jar diff --git a/messaging/gt4.0/java/gar-builder/build.properties b/messaging/gt4.0/java/gar-builder/build.properties index 56a53b72..119d8f42 100644 --- a/messaging/gt4.0/java/gar-builder/build.properties +++ b/messaging/gt4.0/java/gar-builder/build.properties @@ -2,7 +2,7 @@ # General ######### -nimbus.messaging.gt4_0.gar.print-noun=Nimbus GT4.0 main GAR package +nimbus.messaging.gt4_0.gar.print-noun=Nimbus GT4.0 GAR package nimbus.messaging.gt4_0.gar.package.name=nimbus nimbus.messaging.gt4_0.gar.dist.dir=dist nimbus.messaging.gt4_0.gar.build.dir=build @@ -16,6 +16,7 @@ nimbus.basedir=../../../.. nimbus.service.main.basedir=${nimbus.basedir}/service/service/java/source/ nimbus.service.api.basedir=${nimbus.basedir}/service-api/java/source nimbus.authzdbmodule.basedir=${nimbus.basedir}/authzdb +nimbus.autocommon.basedir=${nimbus.basedir}/autocommon nimbus.messaging.gt4_0.basedir=${nimbus.basedir}/messaging/gt4.0/java nimbus.messaging.gt4_0-elastic.basedir=${nimbus.basedir}/messaging/gt4.0-elastic/java nimbus.messaging.query.basedir=${nimbus.basedir}/messaging/query/java/source @@ -72,9 +73,12 @@ nimbus.service.main.share.dir=${nimbus.service.main.basedir}/share # Directories of JARs to package into GAR's LIB directory ######################################################### -nimbus.libservices.dir=../../../../lib/services +nimbus.libservices.dir=../../../../lib/workspaceservice +nimbus.baselibservices.dir=../../../../lib/services nimbus.service.api.dist.dir=${nimbus.service.api.basedir}/dist nimbus.authzdbmodule.dist.dir=${nimbus.authzdbmodule.basedir}/dist +nimbus.autocommon.dist.dir=${nimbus.autocommon.basedir}/dist +nimbus.querygenmodule.dist.dir=${nimbus.basedir}/query/dist nimbus.service.main.dist.dir=${nimbus.service.main.basedir}/dist nimbus.metadata-server.dist.dir=${nimbus.metadata-server.basedir}/dist nimbus.messaging.gt4_0.common.dist.dir=${nimbus.messaging.gt4_0.basedir}/common/dist/ diff --git a/messaging/gt4.0/java/gar-builder/build.xml b/messaging/gt4.0/java/gar-builder/build.xml index a7310b4d..533ed6ec 100644 --- a/messaging/gt4.0/java/gar-builder/build.xml +++ b/messaging/gt4.0/java/gar-builder/build.xml @@ -44,20 +44,24 @@ - - - - - + + + + + + + + + + + - - - + - + + + + + + + + + + + + + + + + + + + @@ -112,11 +134,11 @@ - + - + @@ -144,10 +166,13 @@ COPIES FOR ETC DIRECTORY ******************************************************************* --> - + + + + @@ -158,7 +183,6 @@ - - + diff --git a/metadata/java/source/build.properties b/metadata/java/source/build.properties index cb6b6177..068d5e1d 100644 --- a/metadata/java/source/build.properties +++ b/metadata/java/source/build.properties @@ -7,6 +7,7 @@ nimbus.metadata-server.dist.dir=dist nimbus.metadata-server.jar.name=nimbus-metadata-server.jar nimbus.metadata-server.lib.dir=../../../lib/services +nimbus.metadata-server.lib2.dir=../../../lib/workspaceservice # Compile dependencies from source tree nimbus.service.api.dist.dir=../../../service-api/java/source/dist diff --git a/metadata/java/source/build.xml b/metadata/java/source/build.xml index 7b899d57..409b9729 100644 --- a/metadata/java/source/build.xml +++ b/metadata/java/source/build.xml @@ -52,6 +52,10 @@ + + + + diff --git a/service/service/java/source/build.properties b/service/service/java/source/build.properties index 4135624f..77489908 100644 --- a/service/service/java/source/build.properties +++ b/service/service/java/source/build.properties @@ -6,7 +6,8 @@ nimbus.service.main.src.dir=src nimbus.service.main.dist.dir=dist nimbus.service.main.jar.name=nimbus-workspace-service.jar -nimbus.service.main.lib.dir=../../../../lib/services +nimbus.service.main.baselib.dir=../../../../lib/services +nimbus.service.main.lib.dir=../../../../lib/workspaceservice # Compile dependencies from source tree nimbus.service.api.dist.dir=../../../../service-api/java/source/dist/ diff --git a/service/service/java/source/build.xml b/service/service/java/source/build.xml index d241089e..17dd2bc4 100644 --- a/service/service/java/source/build.xml +++ b/service/service/java/source/build.xml @@ -50,6 +50,9 @@ + + + From a9fc9095699fa228f1ec718998646a2de4fd9bb9 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 14:10:19 -0500 Subject: [PATCH 04/21] Cosmetic things --- .gitignore | 2 ++ authzdb/build.xml | 1 + home/libexec/nimbus_import_users.py | 0 3 files changed, 3 insertions(+) mode change 100644 => 100755 home/libexec/nimbus_import_users.py diff --git a/.gitignore b/.gitignore index 16a11d28..4a623ce7 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ setuptools-0.6c11-py2.7.egg cumulus/deps/boto-2.0b4.tar.gz /cumulus/pip-0.7.2.tar.gz lantorrent/lantorrent.egg-info/ +ctx-broker/gar/etc/other/main.xml + diff --git a/authzdb/build.xml b/authzdb/build.xml index 8993357e..a0ae518c 100644 --- a/authzdb/build.xml +++ b/authzdb/build.xml @@ -59,6 +59,7 @@ classpathref="nimbus.authz.main.classpath" source="1.5" target="1.5" + includeantruntime="false" debug="on"> diff --git a/home/libexec/nimbus_import_users.py b/home/libexec/nimbus_import_users.py old mode 100644 new mode 100755 From b4def1783d4b283ab534671f59548304396a9559 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 14:12:12 -0500 Subject: [PATCH 05/21] Changes to ctx broker for standalone: different authZ and GAR includes libs now --- ctx-broker/build.properties | 1 + ctx-broker/build.xml | 20 ++++-- ctx-broker/gar/build.properties | 6 ++ ctx-broker/gar/build.xml | 56 ++++++++++++++- ...cator.xml => bundled-main.conflocator.xml} | 0 .../etc/other/{main.xml => bundled-main.xml} | 9 +-- ctx-broker/gar/etc/other/standalone-main.xml | 70 +++++++++++++++++++ ctx-broker/gar/etc/user-mapfile | 0 .../ctxbroker/rest/ContextStatus.java | 1 - .../nimbustools/ctxbroker/rest/RestHttp.java | 3 +- ctx-broker/wsdl/stubs/build.xml | 1 + 11 files changed, 149 insertions(+), 18 deletions(-) rename ctx-broker/gar/etc/other/{main.conflocator.xml => bundled-main.conflocator.xml} (100%) rename ctx-broker/gar/etc/other/{main.xml => bundled-main.xml} (92%) create mode 100644 ctx-broker/gar/etc/other/standalone-main.xml create mode 100644 ctx-broker/gar/etc/user-mapfile diff --git a/ctx-broker/build.properties b/ctx-broker/build.properties index cb0a6971..c65dc863 100644 --- a/ctx-broker/build.properties +++ b/ctx-broker/build.properties @@ -18,3 +18,4 @@ nimbus.ctxbroker.lib.dir=../lib/services nimbus.messaging.query.dist.dir=../messaging/query/java/source/dist nimbus.service.api.dist.dir=../service-api/java/source/dist/ nimbus.authz.dist.dir=../authzdb/dist +nimbus.querygeneral.dist.dir=../query/dist diff --git a/ctx-broker/build.xml b/ctx-broker/build.xml index 06a484f9..7bb20af0 100644 --- a/ctx-broker/build.xml +++ b/ctx-broker/build.xml @@ -72,12 +72,12 @@ - + - - + + @@ -168,11 +168,19 @@ GAR RELATED ******************************************************************* --> - - + + + + + + + + + + - + diff --git a/ctx-broker/gar/build.properties b/ctx-broker/gar/build.properties index 5f8e0078..5024df2e 100644 --- a/ctx-broker/gar/build.properties +++ b/ctx-broker/gar/build.properties @@ -30,3 +30,9 @@ nimbusctx.gt4_0.service.dist=../dist # custom build-packages to bundle gar file nimbus.messaging.gt4_0.gar.build.packages.xml=../../scripts/lib/gt4.0/build/build-packages.xml + +nimbusctx.baselibservices.dir=../../lib/services +nimbusctx.service.api.dist.dir=../../service-api/java/source/dist +nimbusctx.authzdbmodule.dist.dir=../../authzdb/dist +nimbusctx.autocommon.dist.dir=../../autocommon/dist +nimbusctx.querygeneral.dist.dir=../../query/dist diff --git a/ctx-broker/gar/build.xml b/ctx-broker/gar/build.xml index 0a914c4a..e69afaf5 100644 --- a/ctx-broker/gar/build.xml +++ b/ctx-broker/gar/build.xml @@ -130,6 +130,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/ctx-broker/gar/etc/other/standalone-main.xml b/ctx-broker/gar/etc/other/standalone-main.xml new file mode 100644 index 00000000..a1f48050 --- /dev/null +++ b/ctx-broker/gar/etc/other/standalone-main.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ctx-broker/gar/etc/user-mapfile b/ctx-broker/gar/etc/user-mapfile new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/src/org/nimbustools/ctxbroker/rest/ContextStatus.java b/ctx-broker/src/org/nimbustools/ctxbroker/rest/ContextStatus.java index 008df62b..bebe4429 100644 --- a/ctx-broker/src/org/nimbustools/ctxbroker/rest/ContextStatus.java +++ b/ctx-broker/src/org/nimbustools/ctxbroker/rest/ContextStatus.java @@ -17,7 +17,6 @@ import org.nimbustools.ctxbroker.blackboard.NodeStatus; -import java.util.ArrayList; import java.util.List; public class ContextStatus { diff --git a/ctx-broker/src/org/nimbustools/ctxbroker/rest/RestHttp.java b/ctx-broker/src/org/nimbustools/ctxbroker/rest/RestHttp.java index 2514e9ba..1caffeff 100644 --- a/ctx-broker/src/org/nimbustools/ctxbroker/rest/RestHttp.java +++ b/ctx-broker/src/org/nimbustools/ctxbroker/rest/RestHttp.java @@ -24,7 +24,6 @@ import org.mortbay.jetty.servlet.Context; import org.mortbay.jetty.servlet.FilterHolder; import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.jetty.webapp.WebAppContext; import org.nimbustools.api.brain.NimbusXmlWebApplicationContext; import org.springframework.web.context.ContextLoaderListener; import org.springframework.web.filter.DelegatingFilterProxy; @@ -141,4 +140,4 @@ public Server getServer() { return server; } -} \ No newline at end of file +} diff --git a/ctx-broker/wsdl/stubs/build.xml b/ctx-broker/wsdl/stubs/build.xml index 5404720a..947e3700 100644 --- a/ctx-broker/wsdl/stubs/build.xml +++ b/ctx-broker/wsdl/stubs/build.xml @@ -39,6 +39,7 @@ ******************************************************************* --> + From dd82e1a6abab67621fdc7c1e60b46fff6aeb1038 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 14:14:08 -0500 Subject: [PATCH 06/21] Separate home/ for standalone broker. Some code dupe at the moment but pynimbusconfig should be what everything uses going foward. --- ctx-broker/home/bin/broker-configure | 38 ++ ctx-broker/home/bin/brokerctl | 28 + ctx-broker/home/lib/ProcessManager.py | 594 ++++++++++++++++++ ctx-broker/home/lib/brokerctl.py | 83 +++ ctx-broker/home/lib/pynimbusconfig/README | 3 + .../home/lib/pynimbusconfig/__init__.py | 0 ctx-broker/home/lib/pynimbusconfig/autoca.py | 273 ++++++++ .../lib/pynimbusconfig/broker/__init__.py | 0 .../home/lib/pynimbusconfig/broker/main.py | 579 +++++++++++++++++ .../home/lib/pynimbusconfig/checkssl.py | 92 +++ ctx-broker/home/lib/pynimbusconfig/ezpz_ca.py | 72 +++ .../home/lib/pynimbusconfig/forcessl.py | 29 + .../home/lib/pynimbusconfig/gtcontainer.py | 133 ++++ .../home/lib/pynimbusconfig/iaas/__init__.py | 0 .../home/lib/pynimbusconfig/iaas/derbyutil.py | 85 +++ .../lib/pynimbusconfig/iaas/groupauthz.py | 169 +++++ .../home/lib/pynimbusconfig/iaas/main.py | 422 +++++++++++++ .../home/lib/pynimbusconfig/javautil.py | 39 ++ .../home/lib/pynimbusconfig/pathutil.py | 194 ++++++ ctx-broker/home/lib/pynimbusconfig/runutil.py | 196 ++++++ .../home/lib/pynimbusconfig/setuperrors.py | 57 ++ .../home/lib/pynimbusconfig/test/__init__.py | 0 .../pynimbusconfig/test/test_groupauthz.py | 73 +++ .../home/lib/pynimbusconfig/web/__init__.py | 0 .../home/lib/pynimbusconfig/web/newconf.py | 95 +++ ctx-broker/home/lib/run-broker.sh | 17 + .../home/lib/run-pynimbusconfig-tests.sh | 22 + 27 files changed, 3293 insertions(+) create mode 100755 ctx-broker/home/bin/broker-configure create mode 100644 ctx-broker/home/bin/brokerctl create mode 100644 ctx-broker/home/lib/ProcessManager.py create mode 100755 ctx-broker/home/lib/brokerctl.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/README create mode 100644 ctx-broker/home/lib/pynimbusconfig/__init__.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/autoca.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/broker/__init__.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/broker/main.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/checkssl.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/ezpz_ca.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/forcessl.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/gtcontainer.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/iaas/__init__.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/iaas/derbyutil.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/iaas/groupauthz.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/iaas/main.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/javautil.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/pathutil.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/runutil.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/setuperrors.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/test/__init__.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/test/test_groupauthz.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/web/__init__.py create mode 100644 ctx-broker/home/lib/pynimbusconfig/web/newconf.py create mode 100644 ctx-broker/home/lib/run-broker.sh create mode 100755 ctx-broker/home/lib/run-pynimbusconfig-tests.sh diff --git a/ctx-broker/home/bin/broker-configure b/ctx-broker/home/bin/broker-configure new file mode 100755 index 00000000..790de1e3 --- /dev/null +++ b/ctx-broker/home/bin/broker-configure @@ -0,0 +1,38 @@ +#!/bin/bash + +PYTHON_EXE="/usr/bin/env python -Wignore::DeprecationWarning" + +NIMBUS_HOME_REL="`dirname $0`/.." +NIMBUS_HOME=`cd $NIMBUS_HOME_REL; pwd` + +if [ -d $NIMBUS_HOME/lib/pynimbusconfig ]; then + NIMBUS_PYLIB=$NIMBUS_HOME/lib +else + echo "Cannot locate Python lib directory" + exit 1 +fi + +PYTHONPATH="$NIMBUS_PYLIB:$PYTHONPATH" +export PYTHONPATH + +# returns 0 if Python 2.5+ +$PYTHON_EXE -c "import sys; sys.exit(sys.version_info < (2,5))" +if [ $? -ne 0 ]; then + echo "ERROR: Your system must have Python version 2.5 or later." + exit 1 +fi + +$PYTHON_EXE $NIMBUS_HOME/lib/pynimbusconfig/broker/main.py --basedir $NIMBUS_HOME $@ +EXITCODE=$? +if [ $EXITCODE -ne 42 ]; then + + if [ $EXITCODE -eq 0 ]; then + exit 0 + else + echo "" + echo "Nimbus is not set up properly, exiting." + exit 2 + fi +fi + + diff --git a/ctx-broker/home/bin/brokerctl b/ctx-broker/home/bin/brokerctl new file mode 100644 index 00000000..2875ca4f --- /dev/null +++ b/ctx-broker/home/bin/brokerctl @@ -0,0 +1,28 @@ +#!/bin/bash + +PYTHON_EXE="/usr/bin/env python -Wignore::DeprecationWarning" + +NIMBUS_HOME_REL="`dirname $0`/.." +NIMBUS_HOME=`cd $NIMBUS_HOME_REL; pwd` + +export NIMBUS_HOME + +if [ -d $NIMBUS_HOME/lib/pynimbusconfig ]; then + NIMBUS_PYLIB=$NIMBUS_HOME/lib +else + echo "Cannot locate Python lib directory" + exit 1 +fi + +PYTHONPATH="$NIMBUS_PYLIB:$PYTHONPATH" +export PYTHONPATH + +# returns 0 if Python 2.5+ +$PYTHON_EXE -c "import sys; sys.exit(sys.version_info < (2,5))" +if [ $? -ne 0 ]; then + echo "ERROR: Your system must have Python version 2.5 or later." + exit 1 +fi + +$PYTHON_EXE $NIMBUS_HOME/lib/brokerctl.py $@ +exit $? diff --git a/ctx-broker/home/lib/ProcessManager.py b/ctx-broker/home/lib/ProcessManager.py new file mode 100644 index 00000000..f310208c --- /dev/null +++ b/ctx-broker/home/lib/ProcessManager.py @@ -0,0 +1,594 @@ +# ---------------------------------------------------------------------------- +# Copyright (c) 2006, Humanized, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of Humanized, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# +# ProcessManager.py +# Author: Atul Varma +# +# Python Version - 2.4 +# +# ---------------------------------------------------------------------------- + +""" + A simple module for process management. Please see the file + README.txt, included with this distribution, for more + information. This file is also available at the following + location: + + http://www.humanized.com/ProcessManager +""" + +# ---------------------------------------------------------------------------- +# TODO's +# +# * Document the public methods better. +# +# * Don't require ProcessManager to be run as root, but do raise +# exceptions if the user tries to control a process that requires +# changing the user ID and it can't be done. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Imports +# ---------------------------------------------------------------------------- + +import os +import sys +import time + + +# ---------------------------------------------------------------------------- +# Public Names and Version Information +# ---------------------------------------------------------------------------- + +__all__ = [ + "Process", + "init", + "add", + "rcScriptMain", + "main" + ] + +__version__ = "0.0.4" + + +# ---------------------------------------------------------------------------- +# Constants +# ---------------------------------------------------------------------------- + +# Amount of time we wait in seconds after starting a process to see if +# it's still alive. +POST_PROCESS_START_DELAY = 5 + +# Amount of time we wait in seconds after killing a process to see if +# it's dead. +POST_PROCESS_STOP_DELAY = 2 + +# A list of all valid commands, accessible from the command-line; they +# map directly to public instance methods of the Process class. +COMMANDS = { + "stop" : "stop the target", + "start" : "start the target", + "restart" : "restart (stop, then start) the target", + "status" : "show status of the target" + } + +# Usage string when running the module's main() function. +USAGE_TEXT = """\ + + %(scriptName)s [options] + +targets: +%(targets)s\ + all (this target applies the command to all + of the above targets) + +commands: +%(commands)s\ +""" + +# Usage string when running the module's rcScriptMain() function. +RC_SCRIPT_USAGE_TEXT = """\ + + %(scriptName)s [options] + +This script controls %(targetDesc)s. + +commands: +%(commands)s\ +""" + +# ---------------------------------------------------------------------------- +# Module Variables +# ---------------------------------------------------------------------------- + +# Directory where all intermediate data files are kept. +_dataDir = None + +# Our process registry; keys are the name identifiers for processes, +# and the values are Process objects. +_processes = {} + +# OptionParser object representing command-line options parser. +_parser = None + +# object storing command-line options, created by an OptionParser +# object. +_options = None + + +# ---------------------------------------------------------------------------- +# Process Class +# ---------------------------------------------------------------------------- + +class Process: + """ + Encapsulates a process that can be stopped, started, and + restarted. + """ + + def __init__( self, + name, + desc, + program, + args, + workingDir, + uid = None, + gid = None, + stopSignal = None, + postStartDelay = POST_PROCESS_START_DELAY): + """ + Creates a process with the given name/identifier, description, + program executable path, argument tuple, and working + directory. When it is run, it will run with the given user + and group ID privileges. When it is stopped, the given signal + will be sent to tell it to do so. + """ + + if stopSignal == None: + import signal + stopSignal = signal.SIGKILL + + self.name = name + self.desc = desc + self.program = program + self.args = [ program ] + self.args.extend( args ) + self.workingDir = workingDir + self.stopSignal = stopSignal + self.postStartDelay = postStartDelay + + if gid: + import grp + self.gid = grp.getgrnam( gid )[2] + else: + self.gid = None + + if uid: + import pwd + self.uid = pwd.getpwnam( uid )[2] + else: + self.uid = None + + def _pidfile( self ): + """ + Returns the filename of the pid file for this process. A pid + file just contains the pid of the process, if it's believed to + be currently running. + """ + + return os.path.join( _dataDir, "%s.pid" % self.name ) + + def _readpid( self ): + """ + Opens the pid file for this process and gets the pid for + it. If the pid file doesn't exist, this method returns None. + """ + + if not os.path.exists( self._pidfile() ): + return None + f = open( self._pidfile(), "r" ) + pid = int( f.read() ) + f.close() + return pid + + def status( self ): + """ + Public method that prints out what this process' status is + (running, stopped, etc). + """ + status = self._getStatus() + print "%-30s%s" % ( self.desc, status ) + if status != 'running': + raise ProcessStatusError() + + def _getStatus( self ): + """ + Returns a single word indicating the status of this process. + """ + + pid = self._readpid() + if pid == None: + return "stopped" + elif _isPidRunning( pid ): + return "running" + else: + return "crashed" + + def start( self, warnCrashed = False ): + """ + Public method that starts the process. If the process is + already deemed to be running, nothing happens. + + If the process fails to launch, raise a + ProcessStartupError exception. + """ + + pid = self._readpid() + if pid != None: + if _isPidRunning( pid ): + print "Process '%s' is already running!" % self.name + return + elif warnCrashed: + print "Process '%s' may have died prematurely." % self.name + + # Start the process now. + leftColumnText = "Launching %s..." % self.desc + print "%-30s" % leftColumnText, + sys.stdout.flush() + + self._doStart() + + def _doStart( self ): + """ + Protected implementation method that starts the actual + process. + """ + + forkResult = os.fork() + if forkResult == 0: + # We're the child process. + + if self.gid: + os.setgid( self.gid ) + + if self.uid: + os.setuid( self.uid ) + + os.chdir( self.workingDir ) + + nullFile = os.open( "/dev/null", os.O_RDWR ) + + # Replace stdin. + os.dup2( nullFile, 0 ) + + # Replace stdout + os.dup2( nullFile, 1 ) + + # Replace stderr + os.dup2( nullFile, 2 ) + + os.close( nullFile ) + + # Launch the program. + os.execv( self.program, self.args ) + else: + # We're the parent process. + pid = forkResult + f = open( self._pidfile(), "w" ) + f.write( "%d" % pid ) + f.close() + + if self.postStartDelay: + time.sleep(self.postStartDelay) + + retVal = os.waitpid( pid, os.WNOHANG ) + if retVal == (0, 0): + print "OK" + else: + print "FAILED" + try: + os.remove(self._pidfile()) + except: + pass + raise ProcessStartupError() + + def stop( self, warnCrashed = True ): + """ + Public method that stops the process if it's currently + running. + """ + + pid = self._readpid() + if pid != None: + if _isPidRunning( pid ): + leftColumnText = "Stopping %s..." % self.desc + print "%-30s" % leftColumnText, + sys.stdout.flush() + + os.kill( pid, self.stopSignal ) + + time.sleep( POST_PROCESS_STOP_DELAY ) + + if not _isPidRunning( pid ): + print "OK" + else: + print "FAILED" + elif warnCrashed: + print "Process '%s' may have died prematurely." % self.name + os.remove( self._pidfile() ) + else: + print "Process '%s' is not running." % self.name + + def restart( self ): + """ + Public method that stops the process and then starts it again. + """ + + self.stop( warnCrashed = False ) + self.start() + +class ProcessStartupError( Exception ): + """ + Exception raised when a process fails to start. + """ + + pass + +class ProcessStatusError( Exception ): + """ + Exception raised when a process is not running. + """ + pass + +# ---------------------------------------------------------------------------- +# Module Functions +# ---------------------------------------------------------------------------- + +def init( dataDir ): + """ + Initializes the module. + + dataDir is the directory where all intermediate data files are + stored (e.g., pidfiles). + """ + + global _dataDir + + _dataDir = dataDir + +def _isPidRunning( pid ): + """ + Returns whether or not a process with the given pid is running. + """ + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + +def add( process ): + """ + Adds the given Process object as a target for the registry of + processes to manage. + """ + + if _processes.has_key( process.name ): + raise TargetAlreadyExistsError() + _processes[process.name] = process + +class TargetAlreadyExistsError( Exception ): + """ + Exception raised when a target is added to the ProcessManager + whose name already exists. + """ + + pass + +def _runCommandOnTarget( command, target ): + """ + Runs the given command on the given target. + """ + + if _dataDir == None: + print "Error! ProcessManager not initialized." + print "Please use ProcessManager.init()." + sys.exit( -1 ) + + errorOccurred = False + + if target == "all": + for process in _processes.values(): + method = getattr( process, command ) + try: + method() + except (ProcessStartupError, ProcessStatusError): + errorOccurred = True + else: + method = getattr( _processes[target], command ) + try: + method() + except (ProcessStartupError, ProcessStatusError): + errorOccurred = True + + if errorOccurred: + sys.exit(1) + +def _checkPrivileges(): + """ + Checks to ensure that the current user has the proper privileges + to run the ProcessManager; exits the program if not. + """ + + needRoot = False + for process in _processes.values(): + if process.gid or process.uid: + needRoot = True + break + + if not needRoot: + return + + if os.getuid() != 0: + print "ERROR: This script must be run as root." + sys.exit( -1 ) + +def _generateTargetHelpText(): + """ + Returns a string containing a list of available targets with their + descriptions. + """ + + targets = "" + for key in _processes.keys(): + targets += " %-21s%s\n" % ( key, _processes[key].desc ) + return targets + +def _generateCommandHelpText(): + """ + Returns a string containing a list of available commands with a + description of what they do. + """ + + commands = "" + for command in COMMANDS.keys(): + commands += " %-21s%s\n" % ( command, COMMANDS[command] ) + commands = commands[:-1] + return commands + +def rcScriptMain(): + """ + The main function of the rc-script use of the Process Manager, + whereby the name of the script determines the target, and the + first command-line parameter determines the command. + """ + + _checkPrivileges() + + target = os.path.split( sys.argv[0] )[1] + if not _processes.has_key( target ): + # If we're in a rc.d directory, we may have 3 characters + # prepended to our name, such as "S01foo". So let's try + # stripping off the first 3 characters of our name and seeing + # if that works as a target. + if target[0] in ["K", "S"]: + ordering = target[1:3] + try: + # See if these characters constitute a number. + int( ordering ) + # If so, let's try reinterpreting our target. + target = target[3:] + except ValueError: + pass + + if not _processes.has_key( target ): + print "ERROR: Target '%s' does not exist!" % target + print "Consider renaming this script to match one" + print "of the following targets:" + print + print _generateTargetHelpText() + sys.exit( -1 ) + + usageTextDict = { + "scriptName" : target, + "targetDesc" : _processes[ target ].desc, + "commands" : _generateCommandHelpText(), + } + + usageText = RC_SCRIPT_USAGE_TEXT % usageTextDict + + _processCmdLineOptions( usageText ) + + if len( sys.argv ) == 1: + command = "" + else: + command = sys.argv[1] + + if not command in COMMANDS.keys(): + _parser.print_help() + sys.exit( -1 ) + + _runCommandOnTarget( command, target ) + +def _processCmdLineOptions( usageText, args=None ): + """ + Parses and processes standard command-line options. + """ + + import optparse + + global _parser + global _options + global _args + + _parser = optparse.OptionParser( usage = usageText ) + + ( _options, _args ) = _parser.parse_args(args=args) + +def main(argv=sys.argv, usage=USAGE_TEXT): + """ + The main function of the Process Manager which processes + command-line arguments and acts on them. + """ + + _checkPrivileges() + + usageTextDict = { + "scriptName" : os.path.split( argv[0] )[1], + "targets" : _generateTargetHelpText(), + "commands" : _generateCommandHelpText(), + } + + usageText = usage % usageTextDict + + _processCmdLineOptions( usageText, args=argv[1:] ) + + if len( _args ) < 2: + _parser.print_help() + sys.exit( -1 ) + + target = _args[0] + command = _args[1] + + if target not in _processes.keys() and target != "all": + print "Invalid target: '%s'" % target + sys.exit( -1 ) + if command not in COMMANDS.keys(): + print "Invalid command: '%s'" % command + sys.exit( -1 ) + + _runCommandOnTarget( command, target ) + diff --git a/ctx-broker/home/lib/brokerctl.py b/ctx-broker/home/lib/brokerctl.py new file mode 100755 index 00000000..5465998f --- /dev/null +++ b/ctx-broker/home/lib/brokerctl.py @@ -0,0 +1,83 @@ +#! /usr/bin/env python + +import os +import sys + +import ProcessManager +from ProcessManager import Process +import ConfigParser + +USAGE_TEXT = """\ + + nimbusctl [target] command + +Omit the target to perform the command for all targets. + +Targets: +%(targets)s\ + +Commands: +%(commands)s\ +""" + + +NIMBUS_HOME = os.getenv("NIMBUS_HOME") + +if not NIMBUS_HOME: + sys.exit("The NIMBUS_HOME environment variable is not set!") + +if not os.path.isdir(NIMBUS_HOME): + sys.exit("$NIMBUS_HOME does not exist: "+ NIMBUS_HOME) + + +CONFIG_PATH = os.path.join(NIMBUS_HOME, 'nimbus-setup.conf') +_NO_CONFIG_ERROR = """ +Could not find the Nimbus setup config file: + %s +This file is created after successful completion of the nimbus-configure +program. You should try running nimbus-configure before using this program. +""" % CONFIG_PATH +config = ConfigParser.SafeConfigParser() +if not config.read(CONFIG_PATH): + sys.exit(_NO_CONFIG_ERROR) +broker_enabled = config.getboolean('nimbussetup', 'broker.enabled') + +if not (broker_enabled): + sys.exit("Broker is not enabled. "+ + "See the '%s' config file to adjust this setting." % CONFIG_PATH) + +try: + services_wait = config.getint('nimbussetup', 'services.wait') +except ConfigParser.NoOptionError: + services_wait = 10 + +NIMBUS_RUN_DIR = os.path.join(NIMBUS_HOME, 'var/run/') +if not os.path.isdir(NIMBUS_RUN_DIR): + try: + os.mkdir(NIMBUS_RUN_DIR) + except: + sys.exit("Failed to create run directory: %s" % NIMBUS_RUN_DIR) + +ProcessManager.init(dataDir = NIMBUS_RUN_DIR) + +if broker_enabled: + NIMBUS_BROKER_EXE = os.path.join(NIMBUS_HOME, 'lib/run-broker.sh') + if not os.path.exists(NIMBUS_BROKER_EXE): + sys.exit("The broker executable does not exist: " + + NIMBUS_BROKER_EXE) + ProcessManager.add( Process( + name = "broker", + desc = "Nimbus Context Broker", + program = NIMBUS_BROKER_EXE, + args = [], + workingDir = NIMBUS_HOME, + postStartDelay=services_wait + )) + + +argv = sys.argv +if len(argv) == 2: + argv = argv[:] + argv.insert(1, 'all') + +ProcessManager.main(argv=argv, usage=USAGE_TEXT) diff --git a/ctx-broker/home/lib/pynimbusconfig/README b/ctx-broker/home/lib/pynimbusconfig/README new file mode 100644 index 00000000..d58b3a79 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/README @@ -0,0 +1,3 @@ +In the next release, the configuration libraries for use with ctx-broker, web, and IaaS will all be housed in a common library. + +This is an interim solution, the future library will be this new "pynimbusconfig" library and it will be accessible from any install's home dir venv (iaas or ctx-broker only). diff --git a/ctx-broker/home/lib/pynimbusconfig/__init__.py b/ctx-broker/home/lib/pynimbusconfig/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/home/lib/pynimbusconfig/autoca.py b/ctx-broker/home/lib/pynimbusconfig/autoca.py new file mode 100644 index 00000000..9609abe3 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/autoca.py @@ -0,0 +1,273 @@ +import os +import shutil +import sys + +from pynimbusconfig import javautil +from pynimbusconfig import pathutil +from pynimbusconfig import runutil +from pynimbusconfig.setuperrors import * + +# Make this False if you want to keep stuff around for examining, otherwise it +# would be in an inconsistent state after an exception during CA creation. +WIPE_NEW_CA_DIRECTORY_ON_ERRORS = True + + +EXE_HOSTGUESS="org.nimbustools.auto_common.HostGuess" +EXE_NEW_HOSTCERTFILE="org.nimbustools.auto_common.confmgr.ReplaceCertFile" +EXE_NEW_HOSTKEYFILE="org.nimbustools.auto_common.confmgr.ReplaceKeyFile" +EXE_CREATE_NEW_CA="org.nimbustools.auto_common.ezpz_ca.GenerateNewCA" +EXE_CREATE_CRL="org.nimbustools.auto_common.ezpz_ca.GenerateCRL" +EXE_CREATE_NEW_CERT="org.nimbustools.auto_common.ezpz_ca.GenerateNewCert" +EXE_FIND_CA_PUBPEM="org.nimbustools.auto_common.ezpz_ca.FindCAPubFile" +EXE_FIND_CA_PRIVPEM="org.nimbustools.auto_common.ezpz_ca.FindCAPrivFile" +EXE_GET_HASHED_CERT_NAME="org.nimbustools.auto_common.ezpz_ca.CertFilenameHash" +EXE_GET_CERT_DN="org.nimbustools.auto_common.ezpz_ca.CertDN" +EXE_WRITE_SIGNING_POLICY="org.nimbustools.auto_common.ezpz_ca.SigningPolicy" +EXE_KEYSTORE_FROM_PEM="org.nimbustools.auto_common.ezpz_ca.KeystoreFromPEM" + +def createCert(CN, basedir, cadir, certtarget, keytarget, log, + allow_overwrite=False): + + if not allow_overwrite and pathutil.check_path_exists(certtarget): + msg = "Certificate file present already: " + certtarget + raise IncompatibleEnvironment(msg) + if not allow_overwrite and pathutil.check_path_exists(keytarget): + msg = "Key file present already: " + keytarget + raise IncompatibleEnvironment(msg) + + cacert_path = findCAcert(basedir, cadir, log) + cakey_path = findCAkey(basedir, cadir, log) + + # Create temp directory. + uuid = pathutil.uuidgen() + tempdir = pathutil.pathjoin(cadir, uuid) + os.mkdir(tempdir) + pathutil.ensure_dir_exists(tempdir, "temp certs directory") + log.debug("Created %s" % tempdir) + + args = [tempdir, CN, "pub", "priv", cacert_path, cakey_path] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_CREATE_NEW_CERT, args=args) + runutil.generic_bailout("Problem creating certificate.", exitcode, stdout, stderr) + + pub_DN = stdout.strip() + + temp_pub_path = pathutil.pathjoin(tempdir, "pub") + pathutil.ensure_file_exists(temp_pub_path, "temp cert") + log.debug("temp cert exists: " + temp_pub_path) + + # copy that to user-cert records + args = [temp_pub_path] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_GET_HASHED_CERT_NAME, args=args) + runutil.generic_bailout("Problem finding hashed cert name.", exitcode, stdout, stderr) + usercertfilehash = stdout.strip() + log.debug("user cert file hash is '%s'" % usercertfilehash) + cert_records_path = pathutil.pathjoin(cadir, "user-certs") + cert_records_path = pathutil.pathjoin(cert_records_path, + usercertfilehash + ".0") + shutil.copyfile(temp_pub_path, cert_records_path) + pathutil.ensure_file_exists(cert_records_path, "new certificate (record)") + log.debug("cert exists at target: " + cert_records_path) + + temp_priv_path = pathutil.pathjoin(tempdir, "priv") + pathutil.ensure_file_exists(temp_priv_path, "temp key") + log.debug("temp key exists: " + temp_priv_path) + + log.debug("Created certificate: %s" % pub_DN) + + # Those user-supplied targets still don't exist, right? :-) + if not allow_overwrite and pathutil.check_path_exists(certtarget): + msg = "Certificate file present already: " + certtarget + raise IncompatibleEnvironment(msg) + if not allow_overwrite and pathutil.check_path_exists(keytarget): + msg = "Key file present already: " + keytarget + raise IncompatibleEnvironment(msg) + + shutil.copyfile(temp_pub_path, certtarget) + pathutil.ensure_file_exists(certtarget, "new certificate") + log.debug("cert exists at target: " + certtarget) + + shutil.copyfile(temp_priv_path, keytarget) + pathutil.ensure_file_exists(keytarget, "new key") + log.debug("key exists at target: " + keytarget) + + pathutil.make_path_rw_private(keytarget) + pathutil.ensure_path_private(keytarget, "new key") + log.debug("file made private: %s" % keytarget) + + shutil.rmtree(tempdir) + + return pub_DN + +class KeystoreMismatchError(Exception): + pass + +def ensureKeystore(certpath, keypath, storepath, password, basedir, log): + """ + Creates or validates a Java keystore from PEM-encoded certificate and key + """ + + if not pathutil.check_path_exists(certpath): + msg = "Certificate file does not exist: " + certpath + raise IncompatibleEnvironment(msg) + + if not pathutil.check_path_exists(keypath): + msg = "Private key file does not exist: " + keypath + raise IncompatibleEnvironment(msg) + + if pathutil.check_path_exists(storepath): + log.debug("Keystore file exists: %s." % storepath, + "Ensuring that it contains right cert/key") + + args = [certpath, keypath, storepath, password] + (exitcode, stdout, stderr) = javautil.run(basedir, log, + EXE_KEYSTORE_FROM_PEM, args=args) + if exitcode == 2: + raise KeystoreMismatchError(stderr) + runutil.generic_bailout("Problem creating keystore", + exitcode, stdout, stderr) + +def getCertDN(certpath, basedir, log): + + if not pathutil.check_path_exists(certpath): + msg = "Certificate file does not exist: " + certpath + raise IncompatibleEnvironment(msg) + + args = [certpath] + (exitcode, stdout, stderr) = javautil.run(basedir, log, + EXE_GET_CERT_DN, args=args) + runutil.generic_bailout("Problem finding cert DN", + exitcode, stdout, stderr) + + return stdout.strip() + +def findCAcert(basedir, cadir, log): + cacertdir = pathutil.pathjoin(cadir, "ca-certs") + args = [cacertdir] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_FIND_CA_PUBPEM, args=args) + runutil.generic_bailout("Problem finding CA certificate.", exitcode, stdout, stderr) + if not stdout: + raise UnexpectedError("Path is not present for CA certificate") + certpath = stdout.strip() + pathutil.ensure_file_exists(certpath, "CA certificate") + return certpath + +def findCAkey(basedir, cadir, log): + cacertdir = pathutil.pathjoin(cadir, "ca-certs") + args = [cacertdir] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_FIND_CA_PRIVPEM, args=args) + runutil.generic_bailout("Problem finding CA key.", exitcode, stdout, stderr) + if not stdout: + raise UnexpectedError("Path is not present for CA key") + keypath = stdout.strip() + pathutil.ensure_file_exists(keypath, "CA key") + return keypath + +def createCA(ca_name, basedir, cadir, log): + if pathutil.check_path_exists(cadir): + raise IncompatibleEnvironment("cannot create a CA at a directory that exists already") + try: + _createCA(ca_name, basedir, cadir, log) + except: + if not WIPE_NEW_CA_DIRECTORY_ON_ERRORS: + raise + # wipe the whole directory + print >>sys.stderr, "Error, wiping the unfinished '%s' directory" % cadir + shutil.rmtree(cadir) + raise + +def _createCA(ca_name, basedir, cadir, log): + + javautil.check(basedir, log) + + # mkdir $cadir + # mkdir $cadir/ca-certs + # mkdir $cadir/trusted-certs + # mkdir $cadir/user-certs + + os.mkdir(cadir) + pathutil.ensure_dir_exists(cadir, "New CA directory") + log.debug("Created %s" % cadir) + + cacertdir = pathutil.pathjoin(cadir, "ca-certs") + os.mkdir(cacertdir) + pathutil.ensure_dir_exists(cacertdir, "New CA certs directory") + log.debug("Created %s" % cacertdir) + + trustedcertdir = pathutil.pathjoin(cadir, "trusted-certs") + os.mkdir(trustedcertdir) + pathutil.ensure_dir_exists(trustedcertdir, "New CA trusted certs directory") + log.debug("Created %s" % trustedcertdir) + + usercertdir = pathutil.pathjoin(cadir, "user-certs") + os.mkdir(usercertdir) + pathutil.ensure_dir_exists(usercertdir, "New CA user certs directory") + log.debug("Created %s" % usercertdir) + + # Create the cert via autocommon + + args = [cacertdir, ca_name] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_CREATE_NEW_CA, args=args) + runutil.generic_bailout("Problem creating CA.", exitcode, stdout, stderr) + + + # Make the private key owner-readable only + + privkeyname = "private-key-" + ca_name + ".pem" + cakeyfile = pathutil.pathjoin(cacertdir, privkeyname) + pathutil.ensure_file_exists(cakeyfile, "New CA key") + log.debug("file exists: %s" % cakeyfile) + pathutil.make_path_rw_private(cakeyfile) + pathutil.ensure_path_private(cakeyfile, "New CA key") + log.debug("file made private: %s" % cakeyfile) + + + # Copy the new certificate file to the "hash.0" version that some toolings + # will expect. + + cacertfile = pathutil.pathjoin(cacertdir, ca_name + ".pem") + pathutil.ensure_file_exists(cacertfile, "New CA cert") + log.debug("file exists: %s" % cacertfile) + + args = [cacertfile] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_GET_HASHED_CERT_NAME, args=args) + runutil.generic_bailout("Problem finding hashed cert name.", exitcode, stdout, stderr) + cacertfilehash = stdout.strip() + log.debug("cert file hash is '%s'" % cacertfilehash) + + newpath = pathutil.pathjoin(cacertdir, cacertfilehash + ".0") + shutil.copyfile(cacertfile, newpath) + pathutil.ensure_file_exists(newpath, "New CA cert (hashed #1)") + log.debug("file exists: %s" % newpath) + + newpath = pathutil.pathjoin(trustedcertdir, cacertfilehash + ".0") + shutil.copyfile(cacertfile, newpath) + pathutil.ensure_file_exists(newpath, "New CA cert (hashed #2)") + log.debug("file exists: %s" % newpath) + + # Signing policy + + signing1 = pathutil.pathjoin(cacertdir, cacertfilehash + ".signing_policy") + args = [cacertfile, signing1] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_WRITE_SIGNING_POLICY, args=args) + runutil.generic_bailout("Problem creating signing_policy file.", exitcode, stdout, stderr) + pathutil.ensure_file_exists(signing1, "signing_policy file #1") + log.debug("file exists: %s" % signing1) + + signing2 = pathutil.pathjoin(trustedcertdir, cacertfilehash + ".signing_policy") + shutil.copyfile(signing1, signing2) + pathutil.ensure_file_exists(signing2, "signing_policy file #2") + log.debug("file exists: %s" % signing2) + + # CRL + + crl1 = pathutil.pathjoin(cacertdir, cacertfilehash + ".r0") + args = [crl1, cacertfile, cakeyfile] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_CREATE_CRL, args=args) + runutil.generic_bailout("Problem creating revocation file.", exitcode, stdout, stderr) + pathutil.ensure_file_exists(crl1, "revocation file #1") + log.debug("file exists: %s" % crl1) + + crl2 = pathutil.pathjoin(trustedcertdir, cacertfilehash + ".r0") + shutil.copyfile(crl1, crl2) + pathutil.ensure_file_exists(crl2, "revocation file #2") + log.debug("file exists: %s" % crl2) diff --git a/ctx-broker/home/lib/pynimbusconfig/broker/__init__.py b/ctx-broker/home/lib/pynimbusconfig/broker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/home/lib/pynimbusconfig/broker/main.py b/ctx-broker/home/lib/pynimbusconfig/broker/main.py new file mode 100644 index 00000000..f71d509e --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/broker/main.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python + +import logging +import optparse +import os +import socket +import sys +import traceback +import ConfigParser +from StringIO import StringIO +import readline +import shutil +import string +import time +from random import Random + +from pynimbusconfig import autoca +from pynimbusconfig import checkssl +from pynimbusconfig import gtcontainer +from pynimbusconfig import javautil +from pynimbusconfig import pathutil +from pynimbusconfig.setuperrors import * + + +CONFIGSECTION = 'nimbussetup' +DEFAULTCONFIG = """ +[nimbussetup] + +# relative to base directory +hostcert: var/hostcert.pem +hostkey: var/hostkey.pem +ca.dir: var/ca +ca.trustedcerts.dir: var/ca/trusted-certs + +gridmap: services/etc/nimbus/nimbus-grid-mapfile + +keystore: var/keystore.jks +keystore.pass: changeit + +services.enabled: True +services.wait: 10 +""" +CONFIG_STATE_PATH = 'nimbus-setup.conf' + +CONFIG_KEY_CA_DIR="ca.dir" +CONFIG_KEY_HOSTCERT="hostcert" +CONFIG_KEY_HOSTKEY="hostkey" +CONFIG_KEY_KEYSTORE="keystore" +CONFIG_KEY_GRIDMAP="gridmap" +CONFIG_KEY_TRUSTED_CERTS="ca.trustedcerts.dir" +CONFIG_KEY_ENVFILE="envfile" + +CA_NAME_QUESTION = """ +Nimbus uses an internal Certificate Authority (CA) for some services. This CA +is also used to generate host and user certificates if you do not have your own. + +This CA will be created in %(ca.dir)s + +Please pick a unique, one word CA name or hit ENTER to use a UUID. + +For example, if you are installing this on the "Jupiter" cluster, you might use +"JupiterNimbusCA" as the name. +""" + +CONFIG_HEADER = """ +# Autogenerated at %(time)s +# +# This file contains configuration values used by the nimbus-configure program. +# If you want to change any of these values, you may edit this file, but you +# must run nimbus-configure before the change will take effect. + +""" + +ENVFILE_BODY = """ +# Autogenerated at %(time)s +# +# This file contains environment variables which are necessary to use some of +# the Nimbus internal tools directly + +NIMBUS_HOME=%(NIMBUS_HOME)s +export NIMBUS_HOME + +GLOBUS_LOCATION=%(GLOBUS_LOCATION)s +export GLOBUS_LOCATION + +X509_CERT_DIR=%(X509_CERT_DIR)s +export X509_CERT_DIR +""" + +KEYSTORE_MISMATCH_MSG = """ +A Java keystore already exists at: + %(keystore)s +However, it does not contain the host certificate and private key which are +being configured. + Certificate: %(hostcert)s + Private key: %(hostkey)s +This may be because you have switched certificates and the keystore contains +the old version. If so, the best solution is to delete (or relocate) the +keystore and rerun nimbus-configure to generate a new one. +""" + +def getlog(override=None): + """Allow developer to replace logging mechanism, e.g. if this + module is incorporated into another program as an API. + + Keyword arguments: + + * override -- Custom logger (default None, uses global variable) + + """ + global _log + if override: + _log = override + try: + _log + except: + _log = logging.getLogger("nimbussetup") + _log.setLevel(logging.DEBUG) + return _log + +def configureLogging(level, formatstring=None, logger=None): + """Configure the logging format and mechanism. Sets global 'log' variable. + + Required parameter: + + * level -- log level + + Keyword arguments: + + * formatstring -- Custom logging format (default None, uses time+level+msg) + + * logger -- Custom logger (default None) + """ + + global log + + logger = getlog(override=logger) + + if not formatstring: + formatstring = "%(asctime)s (%(filename)s:%(lineno)d): %(message)s" + + formatter = logging.Formatter(formatstring) + ch = logging.StreamHandler() + ch.setLevel(level) + ch.setFormatter(formatter) + logger.addHandler(ch) + + # set global variable + log = logger + + log.debug("debug enabled") + +def getconfig(filepaths=None): + config = ConfigParser.SafeConfigParser() + + fh = StringIO(DEFAULTCONFIG) + config.readfp(fh) + if filepaths: + for path in config.read(filepaths): + log.debug("Read config from: '%s'" % path) + return config + +class ARGS: + """Class for command-line argument constants""" + + BASEDIR_LONG = "--basedir" + BASEDIR = "-b" + BASEDIR_HELP = "Path to base Nimbus directory" + + CONFIGPATH_LONG = "--conf" + CONFIGPATH = "-c" + CONFIGPATH_HELP = "Path to configuration file" + + DEBUG_LONG = "--debug" + DEBUG = "-d" + DEBUG_HELP = "Log debug messages" + + HOSTNAME_LONG = "--hostname" + HOSTNAME = "-H" + HOSTNAME_HELP = "Fully qualified hostname of machine" + + CANAME_LONG= "--caname" + CANAME = "-n" + CANAME_HELP = "Unique name to give CA" + + HOSTKEY_LONG = "--hostkey" + HOSTKEY = "-k" + HOSTKEY_HELP = "Path to PEM-encoded host private key" + + HOSTCERT_LONG = "--hostcert" + HOSTCERT = "-C" + HOSTCERT_HELP = "Path to PEM-encoded host certificate" + + PRINT_HOSTNAME_LONG = "--print-hostname" + PRINT_HOSTNAME = "-Z" + PRINT_HOSTNAME_HELP = "Print chosen hostname or error if none chosen" + +def validateargs(opts): + + seeh = "see help (-h)" + + if not opts.basedir: + raise InvalidInput("%s required, %s." % (ARGS.BASEDIR_LONG, seeh)) + + if opts.configpath and not os.path.exists(opts.configpath): + raise InvalidInput("%s file specified does not exist: '%s'" % + (ARGS.CONFIGPATH_LONG, opts.configpath)) + + if opts.hostkey or opts.hostcert: + if not (opts.hostkey and opts.hostcert): + raise InvalidInput( + "You must specify both %s and %s paths, or neither" % + (ARGS.HOSTCERT_LONG, ARGS.HOSTKEY_LONG)) + if not os.path.exists(opts.hostkey): + raise InvalidInput("The specified host key does not exist: %s" % + opts.hostkey) + if not os.path.exists(opts.hostcert): + raise InvalidInput("The specified host cert does not exist: %s" % + opts.hostcert) + +def parsersetup(): + """Return configured command-line parser.""" + + ver = "Nimbus Context Broker setup" + usage = "see help (-h)." + parser = optparse.OptionParser(version=ver, usage=usage) + + group = optparse.OptionGroup(parser, "Misc options", "-------------") + + group.add_option(ARGS.PRINT_HOSTNAME, ARGS.PRINT_HOSTNAME_LONG, + action="store_true", dest="print_chosen_hostname", + default=False, help=ARGS.PRINT_HOSTNAME_HELP) + + group.add_option(ARGS.DEBUG, ARGS.DEBUG_LONG, + action="store_true", dest="debug", default=False, + help=ARGS.DEBUG_HELP) + + group.add_option(ARGS.CONFIGPATH, ARGS.CONFIGPATH_LONG, + dest="configpath", metavar="PATH", + help=ARGS.CONFIGPATH_HELP) + + group.add_option(ARGS.BASEDIR, ARGS.BASEDIR_LONG, + dest="basedir", metavar="PATH", + help=ARGS.BASEDIR_HELP) + + parser.add_option_group(group) + + group = optparse.OptionGroup(parser, "Configuration options", + "-------------") + + group.add_option(ARGS.HOSTNAME, ARGS.HOSTNAME_LONG, + dest="hostname", metavar="HOST", help=ARGS.HOSTNAME_HELP) + + group.add_option(ARGS.CANAME, ARGS.CANAME_LONG, + dest="ca_name", metavar="NAME", help=ARGS.CANAME_HELP) + + group.add_option(ARGS.HOSTKEY, ARGS.HOSTKEY_LONG, + dest="hostkey", metavar="PATH", help=ARGS.HOSTKEY_HELP) + + group.add_option(ARGS.HOSTCERT, ARGS.HOSTCERT_LONG, + dest="hostcert", metavar="PATH", help=ARGS.HOSTCERT_HELP) + parser.add_option_group(group) + return parser + +def fold_opts_to_config(opts, config): + if opts.hostname: + config.set(CONFIGSECTION, 'hostname', opts.hostname) + if opts.ca_name: + config.set(CONFIGSECTION, 'ca.name', opts.ca_name) + if opts.hostkey: + config.set(CONFIGSECTION, CONFIG_KEY_HOSTKEY, opts.hostkey) + if opts.hostcert: + config.set(CONFIGSECTION, CONFIG_KEY_HOSTCERT, opts.hostcert) + +def get_user_input(valuename, default=None, required=True): + answer = None + question = valuename + (default and ("(%s): " % default) or ": ") + while not answer: + value = raw_input(valuename+": ") + if value: + answer = value.strip() + elif default: + answer = default + if not answer: + if required: + print "Invalid input. You must specify a value. Or hit Ctrl-C to give up." + else: + return None + + return answer + +class NimbusSetup(object): + def __init__(self, basedir, config, interactive=True): + self.basedir = basedir + self.config = config + self.interactive = interactive + + self.gtdir = self.resolve_path('services/') + self.cadir = self.resolve_config_path(CONFIG_KEY_CA_DIR) + self.trustedcertsdir = self.resolve_config_path(CONFIG_KEY_TRUSTED_CERTS) + self.hostcert_path = self.resolve_config_path(CONFIG_KEY_HOSTCERT) + self.hostkey_path = self.resolve_config_path(CONFIG_KEY_HOSTKEY) + self.keystore_path = self.resolve_config_path(CONFIG_KEY_KEYSTORE) + self.gridmap_path = self.resolve_config_path(CONFIG_KEY_GRIDMAP) + try: + self.envfile_path = self.resolve_config_path(CONFIG_KEY_ENVFILE) + except: + self.envfile_path = self.resolve_path('libexec/environment.sh') + + def __getitem__(self, key): + try: + return self.config.get(CONFIGSECTION, key) + except ConfigParser.NoOptionError: + return None + + def __setitem__(self, key, value): + return self.config.set(CONFIGSECTION, key, value) + + def validate_environment(self): + if not pathutil.is_absolute_path(self.basedir): + raise IncompatibleEnvironment( + "Base directory setting is not absolute") + pathutil.ensure_dir_exists(self.basedir, "base") + pathutil.ensure_dir_exists(self.gtdir, "GT container") + + # check that we have some java + javautil.check(self.basedir, log) + + def resolve_path(self, path): + """ + Resolves a path relative to base directory. If absolute, returns as-is. + If relative, joins with self.basedir and returns. + """ + if os.path.isabs(path): + return path + return os.path.join(self.basedir, path) + + def resolve_config_path(self, config): + """ + Resolves a path, like resolve_path(), but from a config key. + """ + path = self[config] + if path: + return self.resolve_path(path) + return None + + def is_config_present(self, configkey): + path = self[configkey] + if path: + return True + else: + return False + + def is_config_relative(self, configkey): + """ + Resolves if a config is a relative path or not. + """ + path = self[configkey] + if not path: + return False + return not os.path.isabs(path) + + def ask_hostname(self): + hostguess = self['hostname'] + if not hostguess: + hostguess = socket.getfqdn() + + if self.interactive: + print "\nWhat is the fully qualified hostname of this machine?\n" + print "Press ENTER to use the detected value (%s)\n" % hostguess + hostname = get_user_input("Hostname", default=hostguess) + else: + print "Using hostname: '%s'" % hostguess + hostname = hostguess + return hostname + + def ask_ca_name(self): + ca_name_config = self['ca.name'] + + if self.interactive: + print CA_NAME_QUESTION % {CONFIG_KEY_CA_DIR : self.cadir} + ca_name = get_user_input("CA Name", default=ca_name_config, + required=False) + if not ca_name: + ca_name = pathutil.uuidgen() + print "You did not enter a name, using '%s'" % ca_name + else: + ca_name = ca_name_config or pathutil.uuidgen() + print "Creating CA with name: '%s'" % ca_name + return ca_name + + def write_env_file(self): + """Writes an environment file users can source.""" + f = None + try: + f = open(self.envfile_path,'w') + text = ENVFILE_BODY % { + 'time' : time.strftime('%c'), + 'NIMBUS_HOME' : self.basedir, + 'GLOBUS_LOCATION' : self.gtdir, + 'X509_CERT_DIR' : self.trustedcertsdir + } + f.write(text) + finally: + if f: + f.close() + + def get_hostname_or_ask(self): + if self['hostname']: + hostname = self['hostname'] + log.debug('Using configured hostname: "%s". Run with %s to change.', + hostname, ARGS.HOSTNAME_LONG) + else: + hostname = self.ask_hostname() + self['hostname'] = hostname + return hostname + + def get_hostname_no_asking(self): + # could be None + return self['hostname'] + + def get_repobucket_no_asking(self): + # at least get this to one exact place, can determine dynamically later + return "Repo" + + def perform_setup(self): + # first, set up CA and host cert/key + ca_name = self["ca.name"] + if not os.path.exists(self.cadir): + ca_name = self.ask_ca_name() + self['ca.name'] = ca_name + autoca.createCA(ca_name, self.basedir, self.cadir, log) + if not ca_name: + raise InvalidConfig("CA name is unknown") + + ca_cert = os.path.join(self.cadir, 'ca-certs/%s.pem' % ca_name) + ca_key = os.path.join(self.cadir, 'ca-certs/private-key-%s.pem' % ca_name) + pathutil.ensure_file_exists(ca_cert, "CA certificate") + pathutil.ensure_file_exists(ca_key, "CA private key") + + hostname = self.get_hostname_or_ask() + + #TODO the hostcert/key creation should be extracted from here + # right now it just does a bunch of redundant checks first + checkssl.run(self.basedir, self.hostcert_path, self.hostkey_path, log, + cadir=self.cadir, hostname=hostname) + + password = self['keystore.pass'] + if not password: + raise InvalidConfig("Keystore password is unknown") + + try: + autoca.ensureKeystore(self.hostcert_path, self.hostkey_path, + self.keystore_path, password, self.basedir, log) + except autoca.KeystoreMismatchError: + raise IncompatibleEnvironment(KEYSTORE_MISMATCH_MSG % { + 'keystore' : self.keystore_path, + 'hostcert' : self.hostcert_path, + 'hostkey' : self.hostkey_path }) + pathutil.make_path_rw_private(self.keystore_path) + + # then setup GT container + gtcontainer.adjust_hostname(hostname, self.basedir, self.gtdir, log) + gtcontainer.adjust_secdesc_path(self.basedir, self.gtdir, log) + gtcontainer.adjust_host_cert(self.hostcert_path, self.hostkey_path, + self.basedir, self.gtdir, log) + gtcontainer.adjust_gridmap_file(self.gridmap_path, self.basedir, + self.gtdir, log) + + # and context broker + gtcontainer.adjust_broker_config(ca_cert, ca_key, self.keystore_path, + password, self.basedir, self.gtdir, log) + + # write an enviroment file + self.write_env_file() + +def main(argv=None): + if os.name != 'posix': + print >>sys.stderr, "\nERROR: Only runs on POSIX systems." + return 3 + + if sys.version_info < (2,4): + print >>sys.stderr, "\nERROR: Your system must have Python version 2.4 or later. " + print >>sys.stderr, 'Detected version: "'+sys.version+'"' + return 4 + + parser = parsersetup() + + if argv: + (opts, args) = parser.parse_args(argv[1:]) + else: + (opts, args) = parser.parse_args() + + global log + log = None + + try: + configureLogging(opts.debug and logging.DEBUG or logging.INFO) + + validateargs(opts) + + basedir = opts.basedir + log.debug("base directory: %s" % basedir) + config_state_path = os.path.join(basedir, CONFIG_STATE_PATH) + paths = [config_state_path] + if opts.configpath: + paths.append(opts.configpath) + config = getconfig(filepaths=paths) + #Some command line options are folded into the config object + fold_opts_to_config(opts, config) + + setup = NimbusSetup(basedir, config) + setup.validate_environment() + + if opts.print_chosen_hostname: + hostname = setup.get_hostname_no_asking() + if not hostname: + return 1 + else: + print hostname + return 0 + else: + setup.perform_setup() + + log.debug("saving settings to %s" % config_state_path) + try: + f = None + try: + f = open(config_state_path, 'wb') + f.write(CONFIG_HEADER % {'time' : time.strftime('%c')}) + config.write(f) + except: + log.info("Failed to save settings to %s!" % config_state_path) + finally: + if f: + f.close() + + # using instead of 0 for now, as a special signal to the wrapper program + return 42 + + except InvalidInput, e: + msg = "\nProblem with input: %s" % e.msg + print >>sys.stderr, msg + return 1 + + except InvalidConfig, e: + msg = "\nProblem with configuration: %s" % e.msg + print >>sys.stderr, msg + return 2 + + except IncompatibleEnvironment, e: + msg = "\nCannot validate environment: %s" % e.msg + print >>sys.stderr, msg + if opts.debug: + print >>sys.stderr, "\n---------- stacktrace ----------" + traceback.print_tb(sys.exc_info()[2]) + print >>sys.stderr, "--------------------------------" + return 3 + +if __name__ == "__main__": + try: + sys.exit(main()) + except SystemExit: + raise + except KeyboardInterrupt: + print "\n\nReceived keyboard interrupt. Aborting!\n" + sys.exit(5) + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "\n==> Uncaught problem, please report all following output:\n %s: %s" % (name, err) + print >>sys.stderr, errmsg + traceback.print_tb(sys.exc_info()[2]) + sys.exit(97) diff --git a/ctx-broker/home/lib/pynimbusconfig/checkssl.py b/ctx-broker/home/lib/pynimbusconfig/checkssl.py new file mode 100644 index 00000000..c2801915 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/checkssl.py @@ -0,0 +1,92 @@ +import os +import sys + +from pynimbusconfig import autoca +from pynimbusconfig import pathutil +from pynimbusconfig.setuperrors import * + +def run(basedir, certconf, keyconf, log, cadir=None, hostname=None): + log.debug("Checking SSL") + + # If the configurations themselves are missing, we cannot continue. + if not certconf: + raise IncompatibleEnvironment("There is no 'ssl.cert' configuration") + if not keyconf: + raise IncompatibleEnvironment("There is no 'ssl.key' configuration") + + # If the configurations are relative, they are assumed to be relative from + # the base directory. + if not pathutil.is_absolute_path(certconf): + certconf = pathutil.pathjoin(basedir, certconf) + log.debug("ssl.cert was a relative path, converted to '%s'" % certconf) + if not pathutil.is_absolute_path(keyconf): + keyconf = pathutil.pathjoin(basedir, keyconf) + log.debug("ssl.key was a relative path, converted to '%s'" % keyconf) + + # If the configured certificate exists, check the key permissions, then + # exit. + missingcert = None + missingkey = None + if not pathutil.check_path_exists(certconf): + missingcert = "Configured 'ssl.cert' does not exist at '%s'" % certconf + if not pathutil.check_path_exists(keyconf): + missingkey = "Configured 'ssl.key' does not exist at '%s'" % keyconf + + if not missingcert and not missingkey: + log.debug("cert and key confs exist already, checking key perms") + # check key permission + if pathutil.is_path_private(keyconf): + log.debug("key is owner-read only: %s" % keyconf) + else: + print >>sys.stderr, "***" + print >>sys.stderr, "*** WARNING ***" + print >>sys.stderr, "***" + print >>sys.stderr, "SSL key has bad permissions, should only be readable by the file owner. ssl.key: '%s'" % keyconf + return + + # If only one of the cert/key files exists, we cannot reason about + # what to do: error. + prefix = "Only one of the SSL cert/key file exists, cannot continue. " + if missingcert and not missingkey: + raise IncompatibleEnvironment(prefix + missingcert) + if missingkey and not missingcert: + raise IncompatibleEnvironment(prefix + missingkey) + + + # The configured certificate and key do not exist; create them. + + print "Cannot find configured certificate and key for HTTPS, creating these for you." + + # If the internal CA does not exist, create that first. + if not cadir: + cadir = pathutil.pathjoin(basedir, "var/ca") + if not pathutil.check_path_exists(cadir): + print "\nCannot find internal CA, creating this for you.\n" + print "Please pick a unique, one word CA name or hit return to use a UUID.\n" + print "For example, if you are installing this on the \"Jupiter\" cluster, you could perhaps use \"JupiterNimbusCA\" as the name.\n" + + ca_name = raw_input("Enter a name: ") + + if not ca_name: + ca_name = pathutil.uuidgen() + print "You did not enter a name, using '%s'" % ca_name + else: + ca_name = ca_name.split()[0] + print "Using '%s'" % ca_name + + autoca.createCA(ca_name, basedir, cadir, log) + print "\nCreated internal CA: %s" % cadir + + if not hostname: + print "\nEnter the fully qualified hostname of this machine. If you don't know or care right now, hit return to use 'localhost'.\n" + + hostname = raw_input("Hostname: ") + if not hostname: + hostname = "localhost" + print "Using '%s'" % hostname + + autoca.createCert(hostname, basedir, cadir, certconf, keyconf, log) + print "\nCreated certificate: %s" % certconf + print "Created key: %s\n" % keyconf + + diff --git a/ctx-broker/home/lib/pynimbusconfig/ezpz_ca.py b/ctx-broker/home/lib/pynimbusconfig/ezpz_ca.py new file mode 100644 index 00000000..89dfb63a --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/ezpz_ca.py @@ -0,0 +1,72 @@ +import os +import tempfile + +from pynimbusconfig import autoca + +class EzPzCA(object): + """ + Exposes CA functionality by wrapping java ezpz CA + """ + + def __init__(self, cadir, webdir, tmpdir=None, log=None): + self.cadir = cadir + self.webdir = webdir + self.tmpdir = tmpdir + + if log: + self.log = log + else: + import logging + self.log = logging + + def create_cert(self, cn): + """ + Creates a new certificate with the specified CN. + + Returns a tuple (DN, cert, key) + """ + + (cert_fd, cert_path) = tempfile.mkstemp(dir=self.tmpdir) + (key_fd, key_path) = tempfile.mkstemp(dir=self.tmpdir) + + cert_file = os.fdopen(cert_fd) + key_file = os.fdopen(key_fd) + + try: + dn = autoca.createCert(cn, self.webdir, self.cadir, cert_path, + key_path, self.log, allow_overwrite=True) + + cert = cert_file.read() + key = key_file.read() + + return (dn, cert, key) + + finally: + # best-effort cleanup + + cert_file.close() + key_file.close() + os.remove(cert_path) + os.remove(key_path) + + def get_cert_dn(self, cert): + """ + Determines the DN of a provided certificate. + """ + + (cert_fd, cert_path) = tempfile.mkstemp(dir=self.tmpdir) + + try: + cert_file = os.fdopen(cert_fd, 'wb') + cert_file.write(cert) + cert_file.close() # make sure write is buffered out + + return autoca.getCertDN(cert_path, self.webdir, self.log) + + finally: + try: + os.close(cert_fd) + except: + pass # FD may have been never opened-- or closed above + + os.remove(cert_path) diff --git a/ctx-broker/home/lib/pynimbusconfig/forcessl.py b/ctx-broker/home/lib/pynimbusconfig/forcessl.py new file mode 100644 index 00000000..8cd92b71 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/forcessl.py @@ -0,0 +1,29 @@ +import os +import sys + +from pynimbusconfig import autoca +from pynimbusconfig import pathutil +from pynimbusconfig.setuperrors import * + +def run(basedir, cadir, certconf, keyconf, hostnameconf, log): + log.debug("Forcing a CA/hostcert install") + + # Reject relative paths + if not pathutil.is_absolute_path(cadir): + raise IncompatibleEnvironment("CA directory path is not absolute") + + if not pathutil.is_absolute_path(certconf): + raise IncompatibleEnvironment("certificate path is not absolute") + + if not pathutil.is_absolute_path(keyconf): + raise IncompatibleEnvironment("key path is not absolute") + + # The CA dir must not exist, create that first. + autoca.createCA(pathutil.uuidgen(), basedir, cadir, log) + print "Created auto CA: %s" % cadir + + # The configured certificate and key must not exist; create them. + autoca.createCert(hostnameconf, basedir, cadir, certconf, keyconf, log) + print "\nCreated hostcert: %s" % certconf + print "Created hostkey: %s\n" % keyconf + \ No newline at end of file diff --git a/ctx-broker/home/lib/pynimbusconfig/gtcontainer.py b/ctx-broker/home/lib/pynimbusconfig/gtcontainer.py new file mode 100644 index 00000000..bc562455 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/gtcontainer.py @@ -0,0 +1,133 @@ +from pynimbusconfig import javautil +from pynimbusconfig import pathutil +from pynimbusconfig import runutil +from pynimbusconfig.setuperrors import IncompatibleEnvironment + +EXE_LOGICAL_HOST = "org.nimbustools.auto_common.confmgr.AddOrReplaceLogicalHost" +EXE_PUBLISH_HOST = "org.nimbustools.auto_common.confmgr.AddOrReplacePublishHostname" +EXE_GLOBUS_SECDESC = "org.nimbustools.auto_common.confmgr.AddOrReplaceGlobalSecDesc" +EXE_NEW_GRIDMAPFILE = "org.nimbustools.auto_common.confmgr.ReplaceGridmap" +EXE_NEW_HOSTCERTFILE = "org.nimbustools.auto_common.confmgr.ReplaceCertFile" +EXE_NEW_HOSTKEYFILE = "org.nimbustools.auto_common.confmgr.ReplaceKeyFile" +EXE_SERVICE_RESOURCE = "org.nimbustools.auto_common.confmgr.ServiceResourceAdjust" + +# config paths, relative to $GLOBUS_LOCATION +CONF_SERVERCONFIG = "etc/globus_wsrf_core/server-config.wsdd" +CONF_SECDESC = "etc/globus_wsrf_core/global_security_descriptor.xml" +CONF_BROKERCONFIG = "etc/nimbus-context-broker/jndi-config.xml" + +def adjust_hostname(hostname, basedir, gtdir, log): + serverconfig = get_serverconfig_path(gtdir) + pathutil.ensure_file_exists(serverconfig, "GT server config") + + args = [hostname, serverconfig] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_LOGICAL_HOST, + args=args) + runutil.generic_bailout("Problem adjusting logical host in GT container", + exitcode, stdout, stderr) + + log.debug("Adjusted GT container logical host to %s" % hostname) + + args = ['true', serverconfig] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_PUBLISH_HOST, + args=args) + runutil.generic_bailout("Problem setting GT container to publish hostname", + exitcode, stdout, stderr) + + log.debug("Adjusted GT container to publish hostname in URLs") + +def adjust_gridmap_file(gridmap, basedir, gtdir, log): + if not pathutil.is_absolute_path(gridmap): + raise IncompatibleEnvironment("gridmap path must be absolute") + + pathutil.ensure_file_exists(gridmap, "gridmap") + + secdesc = get_secdesc_path(gtdir) + pathutil.ensure_file_exists(secdesc, "container security settings") + + args = [gridmap, secdesc] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_NEW_GRIDMAPFILE, + args=args) + runutil.generic_bailout("Problem setting new gridmap file location", + exitcode, stdout, stderr) + + log.debug("Adjusted GT container gridmap file to %s" % gridmap) + +def adjust_secdesc_path(basedir, gtdir, log): + + secdesc = get_secdesc_path(gtdir) + pathutil.ensure_file_exists(secdesc, "container security settings") + + serverconfig = get_serverconfig_path(gtdir) + pathutil.ensure_file_exists(serverconfig, "GT server config") + + args = [secdesc, serverconfig] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_GLOBUS_SECDESC, + args=args) + runutil.generic_bailout("Problem activating new security settings in GT container", + exitcode, stdout, stderr) + + log.debug("Activated new security settings file in GT container: %s" % + secdesc) + +def adjust_host_cert(cert, key, basedir, gtdir, log): + + pathutil.ensure_file_exists(cert, "host certificate") + pathutil.ensure_file_exists(key, "host private key") + + secdesc = get_secdesc_path(gtdir) + pathutil.ensure_file_exists(secdesc, "container security settings") + + args = [cert, secdesc] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_NEW_HOSTCERTFILE, + args=args) + runutil.generic_bailout("Problem activating host certificate", + exitcode, stdout, stderr) + log.debug("Activated host certificate file in GT container: %s" % cert) + + args = [key, secdesc] + (exitcode, stdout, stderr) = javautil.run(basedir, log, EXE_NEW_HOSTKEYFILE, + args=args) + runutil.generic_bailout("Problem activating host key", + exitcode, stdout, stderr) + log.debug("Activated host key file in GT container: %s" % cert) + +def adjust_broker_config(cacert, cakey, keystore, keystore_pass, basedir, gtdir, log): + brokerconfig = get_brokerconfig_path(gtdir) + + pathutil.ensure_file_exists(cacert, "CA certificate") + pathutil.ensure_file_exists(cakey, "CA private key") + pathutil.ensure_file_exists(brokerconfig, "Nimbus Context Broker config") + pathutil.ensure_file_exists(keystore, "Java keystore") + + # is some BS + restbroker_xml = pathutil.pathjoin(gtdir, + 'etc/nimbus-context-broker/other/main.xml') + pathutil.ensure_file_exists(restbroker_xml, + "Context Broker REST interface config") + + args = [brokerconfig, 'NimbusContextBroker', 'ctxBrokerBootstrapFactory', + 'caCertPath', cacert, 'caKeyPath', cakey] + (exitcode, stdout, stderr) = javautil.run(basedir, log, + EXE_SERVICE_RESOURCE, args=args) + runutil.generic_bailout("Problem adjusting broker config", + exitcode, stdout, stderr) + + args = [brokerconfig, 'NimbusContextBroker', 'rest', + 'keystoreLocation', keystore, 'keystorePassword', keystore_pass, + 'springConfig', restbroker_xml] + (exitcode, stdout, stderr) = javautil.run(basedir, log, + EXE_SERVICE_RESOURCE, args=args) + runutil.generic_bailout("Problem adjusting broker config", + exitcode, stdout, stderr) + log.debug("Ensured Context Broker CA config: %s" % brokerconfig) + +def get_brokerconfig_path(gtdir): + return pathutil.pathjoin(gtdir, CONF_BROKERCONFIG) + +def get_serverconfig_path(gtdir): + return pathutil.pathjoin(gtdir, CONF_SERVERCONFIG) + +def get_secdesc_path(gtdir): + return pathutil.pathjoin(gtdir, CONF_SECDESC) + diff --git a/ctx-broker/home/lib/pynimbusconfig/iaas/__init__.py b/ctx-broker/home/lib/pynimbusconfig/iaas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/home/lib/pynimbusconfig/iaas/derbyutil.py b/ctx-broker/home/lib/pynimbusconfig/iaas/derbyutil.py new file mode 100644 index 00000000..84beb2a8 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/iaas/derbyutil.py @@ -0,0 +1,85 @@ +import os +import re +import sys +import time +import shutil +import tempfile +from subprocess import Popen, PIPE + +def update_db(ij_path, old_db, new_db): + """ + ij_path -- assumes that this exists + old_db -- path to 2.2 or 2.3 accounting db + new_db -- path to 2.4 accounting db + """ + + def _run_sql_on_db(ij_path, database, sql, user=None, password=None): + script = "connect 'jdbc:derby:%s' " % database + if user and password: + script += "user '%s' " % user + script += "password '%s' " % password + script += ";" + script += sql + script += "disconnect; exit;" + + (script_file, script_filename) = tempfile.mkstemp() + os.write(script_file, script) + os.close(script_file) + + ij_args = ij_path.split() + ij_args.append(script_filename) + output = Popen(ij_args, stdout=PIPE).communicate()[0] + os.remove(script_filename) + + return output + + # Check that the databases exist + if not os.path.exists(old_db): + print >> sys.stderr, "Error in db-update: %s doesn't exist" % old_db + return 1 + + if not os.path.exists(new_db): + print >> sys.stderr, "Error in db-update: %s doesn't exist" % new_db + return 1 + + # determine schema of deployments table + tables_sql = "SHOW TABLES;" + output = _run_sql_on_db(ij_path, old_db, tables_sql) + find_schema = re.compile(".*(APP|NIMBUS)\s*\|DEPLOYMENTS", re.DOTALL) + schema = find_schema.match(output).group(1) + + # Pull data out of old database + select_sql = """ + SELECT uuid, workspaceid, creator_dn, creation_time,\ + requested_duration, active, elapsed_minutes\ + FROM %s.deployments; + """ % schema + output = _run_sql_on_db(ij_path, old_db, select_sql) + if re.match(".*error", output, re.IGNORECASE | re.DOTALL): + print >> sys.stderr, "Error in db-update: Problem getting old data" + print >> sys.stderr, output + return 1 + + + insert_sql = "" + for line in output.splitlines(): + # disgard lines that aren't data + if line.startswith("ij") or line.startswith("UUID ") \ + or line.startswith("-------") or line.endswith("selected")\ + or line == "": + continue + + elements = line.split("|") + elements = [element.strip() for element in elements] + + insert_sql += "INSERT INTO nimbus.deployments (uuid, workspaceid, creator_dn,"\ + "creation_time, requested_duration, active, elapsed_minutes) "\ + "VALUES ('%s', %s, '%s', %s, %s, %s, %s);\n" %\ + (elements[0], elements[1], elements[2], elements[3], + elements[4], elements[5], elements[6]) + + output = _run_sql_on_db(ij_path, new_db, insert_sql) + if re.match(".*error", output, re.IGNORECASE | re.DOTALL): + print >> sys.stderr, "Error in db-update: Problem inserting data" + print >> sys.stderr, output + return 1 diff --git a/ctx-broker/home/lib/pynimbusconfig/iaas/groupauthz.py b/ctx-broker/home/lib/pynimbusconfig/iaas/groupauthz.py new file mode 100644 index 00000000..b21a137d --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/iaas/groupauthz.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +"""Provides basic management of Nimbus groupauthz members. +""" + +import os +import tempfile +from itertools import groupby +import shutil + +_GROUP_FILE_PREFIX = 'group' +_GROUP_PROPS_EXT = '.properties' +_GROUP_MEMBERS_EXT = '.txt' + +def all_groups(groupauthz_dir): + """Returns a list of all valid groups + """ + groups = [] + files = [f for f in os.listdir(groupauthz_dir) + if f.startswith(_GROUP_FILE_PREFIX) and (f.endswith(_GROUP_PROPS_EXT) or f.endswith(_GROUP_MEMBERS_EXT))] + files.sort() + for name, group_files in groupby(files, lambda f: os.path.splitext(f)[0]): + members_path = None + props_path = None + for f in group_files: + ext = os.path.splitext(f)[1].lower() + if ext == _GROUP_PROPS_EXT: + props_path = os.path.join(groupauthz_dir, f) + elif ext == _GROUP_MEMBERS_EXT: + members_path = os.path.join(groupauthz_dir, f) + group_id = int(name[len(_GROUP_FILE_PREFIX):]) + + if members_path and props_path: + groups.append(Group(group_id, props_path, members_path)) + return groups + +def one_group(groupauthz_dir, group_id): + """Returns the specified group, or raises an InvalidGroupError + """ + group_id = _assure_group_id(group_id) + # inefficient, meh + for group in all_groups(groupauthz_dir): + if group.group_id == group_id: + return group + raise InvalidGroupError('group %s was not found' % group_id) + +def group_members(groupauthz_dir, group_id): + """Returns a list of DNs authorized in specified group + """ + group_id = _assure_group_id(group_id) + g = one_group(groupauthz_dir, group_id) + return g.get_members() + +def find_member(groupauthz_dir, dn): + """Finds the group containing a DN, or None + """ + for group in all_groups(groupauthz_dir): + if group.has_member(dn): + return group + return None + +def add_member(groupauthz_dir, dn, group_id=1): + """Adds a DN to a group. if the group already contains DN, do nothing + """ + group_id = _assure_group_id(group_id) + g = one_group(groupauthz_dir, group_id) + return g.add_member(dn) + +def remove_member(groupauthz_dir, dn, group_id=None): + """Removes a DN from all groups, or a specified group + """ + + if group_id: + group_id = _assure_group_id(group_id) + g = one_group(groupauthz_dir, group_id) + return g.remove_member(dn) + removed = False + for group in all_groups(groupauthz_dir): + if group.remove_member(dn): + removed = True + return removed + +def _assure_group_id(group_id): + """Ensures that group ID is an integer + """ + try: + return int(group_id) + except (ValueError, TypeError): + raise InvalidGroupError("Group ID is invalid, must be an integer") + + +class Group(object): + """A single group definition + """ + def __init__(self, group_id, props_path, members_path): + self.group_id = group_id + self.props_path = props_path + self.members_path = members_path + # later we could parse the properties file and grab the group info + + def __str__(self): + return self.group_id + + def get_members(self): + """Returns list of member DNs in this group""" + f = None + members = [] + try: + f = file(self.members_path) + for line in f: + line = line.strip() + if line: + members.append(line) + finally: + if f: f.close() + return members + + def has_member(self, dn): + """Whether specified DN is a member of group + """ + for member in self.get_members(): + if member == dn: + return True + return False + + def add_member(self, dn): + """Add specified DN to group, if it isn't already a member. + Returns update status. + """ + if self.has_member(dn): + return False + f = None + try: + f = file(self.members_path, 'a') + f.write(dn+'\n') + finally: + if f: f.close() + return True + + def remove_member(self, dn): + """Removes specified DN from group. Returns update status. + """ + f = None + tempf = None + try: + removed = False + f = file(self.members_path) + tempfd, temppath = tempfile.mkstemp(text=True) + tempf = os.fdopen(tempfd,'w') + for line in f: + if line.strip() != dn.strip(): + tempf.write(line) + else: + removed = True + f.close() + tempf.close() + shutil.move(temppath, self.members_path) + + return removed + + finally: + if f: f.close() + if tempf: tempf.close() + +class InvalidGroupError(Exception): + """Group does not exist or is invalid + """ + pass + diff --git a/ctx-broker/home/lib/pynimbusconfig/iaas/main.py b/ctx-broker/home/lib/pynimbusconfig/iaas/main.py new file mode 100644 index 00000000..35bd44bc --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/iaas/main.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python + +import ConfigParser +import logging +import optparse +import os +import sys +import traceback + +from pynimbusconfig import checkssl +from pynimbusconfig import forcessl +from pynimbusconfig.web import newconf +from pynimbusconfig import pathutil +from pynimbusconfig.setuperrors import * + +def getlog(override=None): + """Allow developer to replace logging mechanism, e.g. if this + module is incorporated into another program as an API. + + Keyword arguments: + + * override -- Custom logger (default None, uses global variable) + + """ + global _log + if override: + _log = override + try: + _log + except: + _log = logging.getLogger("nimbussetup") + _log.setLevel(logging.DEBUG) + return _log + +def configureLogging(level, formatstring=None, logger=None): + """Configure the logging format and mechanism. Sets global 'log' variable. + + Required parameter: + + * level -- log level + + Keyword arguments: + + * formatstring -- Custom logging format (default None, uses time+level+msg) + + * logger -- Custom logger (default None) + """ + + global log + + logger = getlog(override=logger) + + if not formatstring: + formatstring = "%(asctime)s (%(filename)s:%(lineno)d): %(message)s" + + formatter = logging.Formatter(formatstring) + ch = logging.StreamHandler() + ch.setLevel(level) + ch.setFormatter(formatter) + logger.addHandler(ch) + + # set global variable + log = logger + + log.debug("debug enabled") + +def getconfig(filepath): + if not filepath: + raise InvalidConfig("filepath was not supplied to getconfig()") + config = ConfigParser.SafeConfigParser() + config.read(filepath) + return config + +def config_from_key(config, key, section="nimbusweb"): + setting = config.get(section, key) + log.debug("%s: %s" % (key, setting)) + return setting + +class ARGS: + + """Class for command-line argument constants""" + + BASEDIR_LONG="--basedir" + BASEDIR="-b" + BASEDIR_HELP="Path to base web directory (where bin, lib, etc. is)" + + CHECKSSL_LONG="--checkssl" + CHECKSSL="-s" + CHECKSSL_HELP="Check on SSL setup for standalone server system" + + CONFIGPATH_LONG="--conf" + CONFIGPATH="-c" + CONFIGPATH_HELP="Path to configuration file" + + DEBUG_LONG="--debug" + DEBUG="-d" + DEBUG_HELP="Log debug messages" + + NEWCONF_LONG="--newconf" + NEWCONF="-n" + NEWCONF_HELP="Make sure the internal webapp settings are in line with the standalone system's conf file and filesystem location" + + FORCESSL_LONG="--forcessl" + FORCESSL="-f" + FORCESSL_HELP="Set up a CA and hostcert, useful outside webapp system. \ +It is not interactive." + + FORCECAPATH_LONG="--force-ca-path" + FORCECAPATH="-p" + FORCECAPATH_HELP="Absolute path to non-existent CA directory for %s" % FORCESSL_LONG + + FORCECERTPATH_LONG="--force-hostcert-path" + FORCECERTPATH="-e" + FORCECERTPATH_HELP="Absolute path to non-existent hostcert for %s" % FORCESSL_LONG + + FORCEKEYPATH_LONG="--force-hostkey-path" + FORCEKEYPATH="-k" + FORCEKEYPATH_HELP="Absolute path to non-existent hostkey for %s" % FORCESSL_LONG + + FORCEHOSTNAME_LONG="--force-hostname" + FORCEHOSTNAME="-m" + FORCEHOSTNAME_HELP="hostname to use for hostcert in %s" % FORCESSL_LONG + + PRINTPORT_LONG="--printport" + PRINTPORT="-t" + PRINTPORT_HELP="Print configured port # to stdout" + + PRINTHOST_LONG="--printhost" + PRINTHOST="-l" + PRINTHOST_HELP="Print configured host interface to stdout" + + PRINTCERTPATH_LONG="--printcertpath" + PRINTCERTPATH="-i" + PRINTCERTPATH_HELP="Print configured cert path to stdout and exit" + + PRINTKEYPATH_LONG="--printkeypath" + PRINTKEYPATH="-z" + PRINTKEYPATH_HELP="Print configured key path to stdout and exit" + + # only used by "developer-server.sh" + INSECUREMODE_LONG="--insecuremode" + INSECUREMODE="-a" + INSECUREMODE_HELP="Alter configuration to allow for developer's insecure mode" + +def validateargs(opts): + + actions = [opts.checkssl, opts.forcenewssl, opts.newconf, + opts.printport, opts.printcertpath, opts.printkeypath, + opts.printhost] + + count = 0 + for action in actions: + if action: + count += 1 + + seeh = "see help (-h)" + + if not count: + raise InvalidInput("You must supply an action, %s." % seeh) + + if count != 1: + raise InvalidInput("You may only supply one action, %s." % seeh) + + if not opts.basedir: + raise InvalidInput("%s required, %s." % (ARGS.BASEDIR_LONG, seeh)) + + if opts.forcenewssl: + if not opts.forcecapath: + raise InvalidInput("%s required for %s, %s." % (ARGS.FORCECAPATH_LONG, ARGS.FORCESSL_LONG, seeh)) + if not opts.forcecertpath: + raise InvalidInput("%s required for %s, %s." % (ARGS.FORCECERTPATH_LONG, ARGS.FORCESSL_LONG, seeh)) + if not opts.forcekeypath: + raise InvalidInput("%s required for %s, %s." % (ARGS.FORCEKEYPATH_LONG, ARGS.FORCESSL_LONG, seeh)) + if not opts.forcehostname: + raise InvalidInput("%s required for %s, %s." % (ARGS.FORCEHOSTNAME_LONG, ARGS.FORCESSL_LONG, seeh)) + +def parsersetup(): + """Return configured command-line parser.""" + + ver="Nimbus webapp setup" + usage="see help (-h)." + parser = optparse.OptionParser(version=ver, usage=usage) + + # ---- + + group = optparse.OptionGroup(parser, "Misc options", "-------------") + + group.add_option(ARGS.DEBUG, ARGS.DEBUG_LONG, + action="store_true", dest="debug", default=False, + help=ARGS.DEBUG_HELP) + + group.add_option(ARGS.CONFIGPATH, ARGS.CONFIGPATH_LONG, + dest="configpath", metavar="PATH", + help=ARGS.CONFIGPATH_HELP) + + group.add_option(ARGS.BASEDIR, ARGS.BASEDIR_LONG, + dest="basedir", metavar="PATH", + help=ARGS.BASEDIR_HELP) + + group.add_option(ARGS.INSECUREMODE, ARGS.INSECUREMODE_LONG, + action="store_true", dest="insecuremode", default=False, + help=ARGS.INSECUREMODE_HELP) + + parser.add_option_group(group) + + # ---- + + group = optparse.OptionGroup(parser, + "Required action, one is required", + "-------------------------------") + + group.add_option(ARGS.CHECKSSL, ARGS.CHECKSSL_LONG, + action="store_true", dest="checkssl", default=False, + help=ARGS.CHECKSSL_HELP) + + group.add_option(ARGS.PRINTPORT, ARGS.PRINTPORT_LONG, + action="store_true", dest="printport", default=False, + help=ARGS.PRINTPORT_HELP) + + group.add_option(ARGS.PRINTHOST, ARGS.PRINTHOST_LONG, + action="store_true", dest="printhost", default=False, + help=ARGS.PRINTHOST_HELP) + + group.add_option(ARGS.PRINTCERTPATH, ARGS.PRINTCERTPATH_LONG, + action="store_true", dest="printcertpath", default=False, + help=ARGS.PRINTCERTPATH_HELP) + + group.add_option(ARGS.PRINTKEYPATH, ARGS.PRINTKEYPATH_LONG, + action="store_true", dest="printkeypath", default=False, + help=ARGS.PRINTKEYPATH_HELP) + + group.add_option(ARGS.NEWCONF, ARGS.NEWCONF_LONG, + action="store_true", dest="newconf", default=False, + help=ARGS.NEWCONF_HELP) + + group.add_option(ARGS.FORCESSL, ARGS.FORCESSL_LONG, + action="store_true", dest="forcenewssl", default=False, + help=ARGS.FORCESSL_HELP) + + parser.add_option_group(group) + + # ---- + + group = optparse.OptionGroup(parser, + "Required options for forcessl", + "----------------------------") + + group.add_option(ARGS.FORCECAPATH, ARGS.FORCECAPATH_LONG, + dest="forcecapath", metavar="DIR", + help=ARGS.FORCECAPATH_HELP) + + group.add_option(ARGS.FORCECERTPATH, ARGS.FORCECERTPATH_LONG, + dest="forcecertpath", metavar="PATH", + help=ARGS.FORCECERTPATH_HELP) + + group.add_option(ARGS.FORCEKEYPATH, ARGS.FORCEKEYPATH_LONG, + dest="forcekeypath", metavar="PATH", + help=ARGS.FORCEKEYPATH_HELP) + + group.add_option(ARGS.FORCEHOSTNAME, ARGS.FORCEHOSTNAME_LONG, + dest="forcehostname", metavar="HOSTNAME", + help=ARGS.FORCEHOSTNAME_HELP) + + parser.add_option_group(group) + + # ---- + + return parser + +def main(argv=None): + if os.name != 'posix': + print >>sys.stderr, "Only runs on POSIX systems." + return 3 + + parser = parsersetup() + + if argv: + (opts, args) = parser.parse_args(argv[1:]) + else: + (opts, args) = parser.parse_args() + + global log + log = None + + printdebugoutput = False + + try: + + # 1. Intake args and confs + + validateargs(opts) + config = getconfig(filepath=opts.configpath) + + # 2. Setup logging + + confdebug = config.get("nimbusweb", "debug") + if confdebug == "on": + printdebugoutput = True + elif opts.debug: + printdebugoutput = True + + if printdebugoutput: + configureLogging(logging.DEBUG) + else: + configureLogging(logging.INFO) + + # 3. Dump settings + + basedir = opts.basedir + log.debug("base directory: %s" % basedir) + + insecuremode = opts.insecuremode + if insecuremode: + log.debug("**** This is insecure developer mode ****") + else: + log.debug("secure mode") + + certconf = config_from_key(config, "ssl.cert") + keyconf = config_from_key(config, "ssl.key") + cadir = config_from_key(config, "ca.dir") + timezone = config_from_key(config, "timezone") + port = config_from_key(config, "webserver.port") + host = config_from_key(config, "webserver.host") + printurl = config_from_key(config, "print.url") + accountprompt = config_from_key(config, "account.prompt") + expire_hours = config_from_key(config, "token.expire_hours") + try: + expire_hours = int(expire_hours) + except: + raise InvalidConfig("invalid token.expire_hours setting, not an integer?") + + # 4. Validate base directory + + if not pathutil.is_absolute_path(basedir): + raise IncompatibleEnvironment("Base directory setting is not absolute, have you been altering the stanadalone launch code?") + + pathutil.ensure_dir_exists(basedir, "base", ": have you been altering the stanadalone launch code?") + + # 5. Run one subcommand + + if opts.checkssl: + checkssl.run(basedir, certconf, keyconf, log) + + if opts.newconf: + newconf.run(basedir, timezone, accountprompt, log, + printdebugoutput, insecuremode, printurl, expire_hours, + cadir) + + if opts.printport: + if not port: + raise IncompatibleEnvironment("There is no 'webserver.port' configuration") + try: + port = int(port) + except: + raise IncompatibleEnvironment("'webserver.port' configuration is not an integer?") + print port + + if opts.printhost: + if not host: + raise IncompatibleEnvironment("There is no 'webserver.host' configuration") + print host + + if opts.printcertpath: + if not certconf: + raise IncompatibleEnvironment("There is no 'ssl.cert' configuration") + if not pathutil.is_absolute_path(certconf): + certconf = pathutil.pathjoin(basedir, certconf) + log.debug("ssl.cert was a relative path, converted to '%s'" % certconf) + print certconf + + if opts.printkeypath: + if not keyconf: + raise IncompatibleEnvironment("There is no 'ssl.key' configuration") + if not pathutil.is_absolute_path(keyconf): + keyconf = pathutil.pathjoin(basedir, keyconf) + log.debug("ssl.key was a relative path, converted to '%s'" % keyconf) + print keyconf + + if opts.forcenewssl: + forcessl.run(basedir, opts.forcecapath, opts.forcecertpath, + opts.forcekeypath, opts.forcehostname, log) + + except InvalidInput, e: + msg = "\nProblem with input: %s" % e.msg + print >>sys.stderr, msg + return 1 + + except InvalidConfig, e: + msg = "\nProblem with configuration: %s" % e.msg + print >>sys.stderr, msg + return 2 + + except IncompatibleEnvironment, e: + msg = "\nCannot validate environment: %s" % e.msg + print >>sys.stderr, msg + if printdebugoutput: + print >>sys.stderr, "\n---------- stacktrace ----------" + traceback.print_tb(sys.exc_info()[2]) + print >>sys.stderr, "--------------------------------" + return 3 + +if __name__ == "__main__": + + try: + sys.exit(main()) + except SystemExit: + raise + except KeyboardInterrupt: + raise + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "\n==> Uncaught problem, please report all following output:\n %s: %s" % (name, err) + print >>sys.stderr, errmsg + traceback.print_tb(sys.exc_info()[2]) + sys.exit(97) + diff --git a/ctx-broker/home/lib/pynimbusconfig/javautil.py b/ctx-broker/home/lib/pynimbusconfig/javautil.py new file mode 100644 index 00000000..71e43ba6 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/javautil.py @@ -0,0 +1,39 @@ +import os +from pynimbusconfig import pathutil +from pynimbusconfig import runutil +from pynimbusconfig.setuperrors import IncompatibleEnvironment + +EXE_JVMCHECK="org.nimbustools.auto_common.JVMCheck" + +def check(basedir, log): + (exitcode, stdout, stderr) = run(basedir, log, EXE_JVMCHECK) + if exitcode != 0: + msg = "Java problem:\n" + stderr + "\n" + if stdout: + msg += "(stdout '%s')" % stdout + raise IncompatibleEnvironment(msg) + +def run(basedir, log, klass, args=[]): + cmd = "java -classpath %s" % classpath(basedir) + cmd += " " + klass + for arg in args: + cmd += " \"" + arg + "\"" + return runutil.runexe(cmd, log, killtime=0) + +def jarsdirs(basedir): + jarsdirs = [] + for dirname in ["lib/java", "services/lib"]: + d = pathutil.pathjoin(basedir, dirname) + if os.path.exists(d): + jarsdirs.append(d) + return jarsdirs + +def classpath(basedir): + opt = "." + libdirs = jarsdirs(basedir) + for libdir in libdirs: + fnames = os.listdir(libdir) + for fname in fnames: + if fname.endswith(".jar"): + opt += ":" + pathutil.pathjoin(libdir, fname) + return opt diff --git a/ctx-broker/home/lib/pynimbusconfig/pathutil.py b/ctx-broker/home/lib/pynimbusconfig/pathutil.py new file mode 100644 index 00000000..cc0c82de --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/pathutil.py @@ -0,0 +1,194 @@ +import commands +import os +import stat +import sys +import uuid +from commands import getstatusoutput + +from pynimbusconfig.setuperrors import * + +def ensure_file_exists(path, name, extramsg=None): + _ensure_exists(path, name, extramsg, "file") + +def ensure_dir_exists(path, name, extramsg=None): + _ensure_exists(path, name, extramsg, "directory") + +def _ensure_exists(path, name, extramsg, ptype): + if not check_path_exists(path): + msg = "%s %s '%s' does not exist" % (name, ptype, path) + if extramsg: + msg += extramsg + raise IncompatibleEnvironment(msg) + + adir = os.path.isdir(path) + afile = not adir + if ptype == "file": + if not afile: + msg = "%s %s '%s' is not a file" % (name, ptype, path) + raise IncompatibleEnvironment(msg) + elif ptype == "directory": + if not adir: + msg = "%s %s '%s' is not a directory" % (name, ptype, path) + raise IncompatibleEnvironment(msg) + else: + raise IncompatibleEnvironment("Unknown path type '%s'" % ptype) + +def check_path_exists(path): + return os.path.exists(path) + +def is_absolute_path(path): + """works on abstract paths, they do not need to exist""" + return os.path.isabs(path) + +def pathjoin(above, below): + if os.path.isabs(below): + raise InvalidInput("Absolute paths on the 'right side' produce unexpected result") + return os.path.join(above, below) + +def uuidgen(): + return str(uuid.uuid4()) + +def make_path_rw_private(path): + mode = stat.S_IRUSR | stat.S_IWUSR + os.chmod(path, mode) + +def ensure_path_private(path, name): + if not is_path_private(path): + raise IncompatibleEnvironment("%s expected to be private: %s" % (name, path)) + +def is_path_private(path): + statresult = os.stat(path) + midx = stat.ST_MODE + mode = statresult[midx] + mode = stat.S_IMODE(mode) + if not mode & stat.S_IRUSR: + return False + for i in ("GRP", "OTH"): + for perm in "R", "W", "X": + if mode & getattr(stat, "S_I"+ perm + i): + return False + return True + +def modeStr(mode): + string="" + mode=stat.S_IMODE(mode) + for i in ("USR", "GRP", "OTH"): + for perm in "R", "W", "X": + if mode & getattr(stat, "S_I"+ perm + i): + string = string + perm.lower() + else: + string = string + "-" + return string + +def mode600(mode): + mode = stat.S_IMODE(mode) + if not mode & stat.S_IRUSR: + return False + if not mode & stat.S_IWUSR: + return False + for i in ("GRP", "OTH"): + for perm in "R", "W", "X": + if mode & getattr(stat, "S_I"+ perm + i): + return False + return True + +def write_repl_file(path, outputtext, log): + """TODO: switch this to use tempfile.mkstemp""" + f = None + try: + try: + # touch the file or replace what was there + f = open(path, 'w') + f.close() + f = None + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "Problem creating '%s': %s: %s\n" \ + % (path, name, err) + log.error(errmsg) + raise UnexpectedError(errmsg) + finally: + if f: + f.close() + + # chmod user-only read/write + target = stat.S_IRUSR | stat.S_IWUSR + try: + os.chmod(path, target) + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "Problem chmod-ing '%s': %s: %s\n" % (path, name, err) + #log.error(errmsg) + raise UnexpectedError(errmsg) + + # make sure it happened + midx = stat.ST_MODE + errmsg = "Failed to modify '%s' to %s" % (path, modeStr(target)) + en2 = os.stat(path) + if not mode600(en2[midx]): + raise UnexpectedError(errmsg) + + log.debug("Created '%s' and modified permissions to 600" % path) + + f = None + try: + try: + f = open(path, 'w') + f.write(outputtext) + f.write("\n") + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "Problem writing to '%s': %s: %s\n" \ + % (path, name, err) + #log.error(errmsg) + raise UnexpectedError(errmsg) + finally: + if f: + f.close() + + log.debug("Wrote '%s'." % path) + +# from Python 2.6 standard library, slightly modified +def _relpath(path, start): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + if not start: + raise ValueError("no start specified") + + start_list = os.path.abspath(start).split(os.path.sep) + path_list = os.path.abspath(path).split(os.path.sep) + + # Work out how much of the filepath is shared by start and path. + i = len(os.path.commonprefix([start_list, path_list])) + + rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return curdir + return os.path.join(*rel_list) + +relpath = None +try: + # os.path.relpath is in 2.6+ + relpath = getattr(os.path, 'relpath') +except AttributeError: + relpath = _relpath diff --git a/ctx-broker/home/lib/pynimbusconfig/runutil.py b/ctx-broker/home/lib/pynimbusconfig/runutil.py new file mode 100644 index 00000000..0220bbaf --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/runutil.py @@ -0,0 +1,196 @@ +import os +import signal +import sys +import time + +from pynimbusconfig.setuperrors import * + +try: + from threading import Thread +except ImportError: + from dummy_threading import Thread + +from commands import getstatusoutput + +# not using subprocess package to maintain at least python 2.3 compat. +from popen2 import Popen3 + +def generic_bailout(header, exitcode, stdout, stderr): + if exitcode != 0: + # header something like "Problem creating CA." + msg = "%s\n" % header + if stdout: + msg += "[<<< stdout: '%s'\n" % stdout + if stderr: + msg += "[<<< stderr: '%s'\n" % stderr + raise UnexpectedError(msg) + +class SimpleRunThread(Thread): + """Run a command with timeout options, delay, stdin, etc.""" + + def __init__ (self, cmd, log, killsig=-1, killtime=0, stdin=None, delay=None): + """Populate the thread. + + Required parameters: + + * cmd -- command to run + + * log -- logger + + Keyword parameters: + + * killsig -- signum to kill with, default is unset + (needed if you set a killtime) + + * killtime -- secs (float or int) to wait before kill, default is + unset (if set, needs killsig parameter) + + * stdin -- optional stdin to push, default is unset + + * delay -- secs (float or int) to wait before invoking cmd + + Properties available: + + * stdout -- stdout data or None + + * stderr -- stderr data or None + + * killed -- boolean, set True if cmd was killed + + * exception -- if kill won't work + + """ + + Thread.__init__(self) + self.cmd = cmd + self.log = log + self.stdin = stdin + self.killsig = killsig + self.killtime = float(killtime) + self.delay = delay + self.exception = None + self.exit = None + self.stdout = None + self.stderr = None + self.killed = False + + def run(self): + if self.delay: + self.log.debug("delaying for %.3f secs: '%s'" % (self.delay, self.cmd)) + time.sleep(self.delay) + self.log.debug("program starting '%s'" % self.cmd) + p = Popen3(self.cmd, True) + if self.stdin: + if p.poll() == -1: + p.tochild.write(self.stdin) + p.tochild.flush() + p.tochild.close() + #log.debug("wrote '%s' to child" % self.stdin) + else: + self.log.error("child exited before stdin was written to") + done = False + while not done and self.killtime > 0: + time.sleep(0.2) + if p.poll() != -1: + done = True + self.killtime -= 0.2 + + if not done and self.killsig != -1: + try: + os.kill(p.pid, self.killsig) + self.killed = True + except OSError, e: + self.log.exception("problem killing") + self.exception = e + return + + self.exit = p.wait() >> 8 + self.stdout = p.fromchild.read() + self.stderr = p.childerr.read() + p.fromchild.close() + p.childerr.close() + self.log.debug("program ended: '%s'" % self.cmd) + +def runexe(cmd, log, killtime=2.0): + """Run a system program. + + Required parameter: + + * cmd -- command to run, string + + * log -- logger + + Keyword parameter: + + * killtime -- how many seconds to wait before SIGKILL (int or float) + Default is 2.0 seconds. + + Return (exitcode, stdout, stderr) + + * exitcode -- integer exit code + + * stdout -- stdout or None + + * stderr -- stderr or None + + Raises IncompatibleEnvironment for serious issue (but not on non-zero exit) + + """ + + if killtime > 0: + thr = SimpleRunThread(cmd, log, killsig=signal.SIGKILL, killtime=killtime) + else: + thr = SimpleRunThread(cmd, log) + thr.start() + thr.join() + + # sudo child won't take signals + if thr.exception: + raise IncompatibleEnvironment(str(thr.exception)) + + return (thr.exit, thr.stdout, thr.stderr) + +def getstdout(cmd, strip=False): + """Retrieve stdout simply. + + Required parameter: + + * cmd -- single string command to run. + + Keyword parameters: + + * strip -- remove last newline if it exists (boolean, default is False) + + Raises IncompatibleEnvironment if there is a problem (defined by presence + of stderr...). + + Return stdout. + + """ + + try: + fstdin, fstdout, fstderr = os.popen3(cmd) + fstdin.close() + stdout = fstdout.read() + fstdout.close() + stderr = fstderr.read() + fstderr.close() + except: + exception_type = sys.exc_type + try: + exceptname = exception_type.__name__ + except AttributeError: + exceptname = exception_type + name = str(exceptname) + err = str(sys.exc_value) + errmsg = "Problem running '%s': %s: %s\n" % (cmd, name, err) + raise IncompatibleEnvironment(errmsg) + if stderr: + raise IncompatibleEnvironment(stderr) + else: + if stdout: + if strip and stdout[-1] == '\n': + stdout = stdout[:-1] + return stdout + else: + raise IncompatibleEnvironment("no output?") diff --git a/ctx-broker/home/lib/pynimbusconfig/setuperrors.py b/ctx-broker/home/lib/pynimbusconfig/setuperrors.py new file mode 100644 index 00000000..599ac345 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/setuperrors.py @@ -0,0 +1,57 @@ + +class InvalidInput(Exception): + + """Exception for illegal commandline syntax/combinations.""" + + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class InvalidConfig(Exception): + + """Exception for misconfigurations.""" + + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class IncompatibleEnvironment(Exception): + + """Exception for when something has determined a problem with the + deployment environment.""" + + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class UnexpectedError(Exception): + + """Exception for when the program cannot proceed.""" + + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + +class CLIError(Exception): + """An exception for nimbus command line tools""" + + e_code = {} + e_code['EUSER'] = 1 + e_code['ENIMBUSHOME'] = 2 + e_code['EPATH'] = 3 + e_code['ECMDLINE'] = 4 + + def __init__(self, type, msg): + self.type = type + self.msg = msg + self.rc = CLIError.e_code[type] + + def get_rc(self): + return self.rc + + def __str__(self): + return self.msg diff --git a/ctx-broker/home/lib/pynimbusconfig/test/__init__.py b/ctx-broker/home/lib/pynimbusconfig/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/home/lib/pynimbusconfig/test/test_groupauthz.py b/ctx-broker/home/lib/pynimbusconfig/test/test_groupauthz.py new file mode 100644 index 00000000..5548c410 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/test/test_groupauthz.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import unittest +import tempfile +import shutil +import uuid +import os + +from pynimbusconfig.iaas import groupauthz + +class GroupauthzTests(unittest.TestCase): + def setUp(self): + self._dir = tempfile.mkdtemp() + + def write_groups(self, groups): + for group_id, members in groups.iteritems(): + base = os.path.join(self._dir, + groupauthz._GROUP_FILE_PREFIX+ '%02d' % group_id) + props_file = file(base+groupauthz._GROUP_PROPS_EXT, 'w') + props_file.close() + members_file = file(base+groupauthz._GROUP_MEMBERS_EXT, 'w') + for member in members: + members_file.write(member+'\n') + members_file.close() + + def test_basic_ops(self): + + groups = { + 1:[new_dn() for i in range(5)], + 2:[new_dn() for i in range(3)], + 3:[], + 4:[new_dn() for i in range(1)]} + self.write_groups(groups) + + #and throw in an unrelated file + file(os.path.join(self._dir,'README'),'w').close() + + found_groups = groupauthz.all_groups(self._dir) + self.assertEqual(len(groups),len(found_groups)) + + ids = set([g.group_id for g in found_groups]) + self.assertEqual(len(groups),len(ids)) + for id in ids: + self.assertTrue(id in groups) + + members = groupauthz.group_members(self._dir, 1) + self.assertEqual(len(groups[1]), len(members)) + self.assertFalse(groupauthz.add_member(self._dir, groups[1][0])) + new_member = new_dn() + self.assertTrue(groupauthz.add_member(self._dir, new_member)) + groups[1] = groupauthz.group_members(self._dir, 1) + self.assertEqual(len(groups[1]), len(members)+1) + self.assertFalse(groupauthz.add_member(self._dir, new_member)) + self.assertEqual(len(groups[1]), + len(groupauthz.group_members(self._dir, 1))) + + to_remove = groups[1][2] + self.assertTrue(groupauthz.remove_member(self._dir, to_remove)) + members = groupauthz.group_members(self._dir, 1) + self.assertFalse(to_remove in members) + self.assertEqual(len(members), len(groups[1])-1) + + def tearDown(self): + if self._dir: + shutil.rmtree(self._dir) + + +def new_dn(): + return str(uuid.uuid4()) + +if __name__ == '__main__': + unittest.main() + diff --git a/ctx-broker/home/lib/pynimbusconfig/web/__init__.py b/ctx-broker/home/lib/pynimbusconfig/web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ctx-broker/home/lib/pynimbusconfig/web/newconf.py b/ctx-broker/home/lib/pynimbusconfig/web/newconf.py new file mode 100644 index 00000000..f208c538 --- /dev/null +++ b/ctx-broker/home/lib/pynimbusconfig/web/newconf.py @@ -0,0 +1,95 @@ +from random import Random +import string + +from pynimbusconfig import pathutil +from pynimbusconfig.setuperrors import * + +def run(basedir, timezone, accountprompt, log, debug, insecuremode, printurl, + expire_hours, cadir): + log.debug("Installing new configurations to django and cherrypy") + + if not accountprompt: + accountprompt = "contact the administrator." + + if not timezone: + raise IncompatibleEnvironment("There is no 'timezone' configuration") + + # -------------------------------------------------------------------------- + # The generated_settings.py file is created and replaced at will by this + # newconf system. + + # sanity check: + real_settings = pathutil.pathjoin(basedir, "src/python/nimbusweb/portal/settings.py") + pathutil.ensure_file_exists(real_settings, "web settings") + log.debug("file exists: %s" % real_settings) + + generated_settings = pathutil.pathjoin(basedir, "src/python/nimbusweb/portal/generated_settings.py") + if pathutil.check_path_exists(generated_settings): + log.debug("Going to overwrite previously written generated_settings.py") + + lines = [] + + # sqlite DB + db_path = pathutil.pathjoin(basedir, "var/nimbus.sqlite") + lines.append("DATABASE_ENGINE = 'sqlite3'") + lines.append("DATABASE_NAME = '%s'" % db_path) + + lines.append("TIME_ZONE = '%s'" % timezone) + lines.append("NIMBUS_ACCOUNT_PROMPT = '%s'" % accountprompt) + + cadir_path = pathutil.pathjoin(basedir, cadir) + lines.append("NIMBUS_CADIR = '%s'" % cadir_path) + + if debug: + lines.append("DEBUG = True") + lines.append("TEMPLATE_DEBUG = True") + else: + lines.append("DEBUG = False") + lines.append("TEMPLATE_DEBUG = False") + + if insecuremode: + lines.append("SESSION_COOKIE_SECURE = False") + else: + lines.append("SESSION_COOKIE_SECURE = True") + + lines.append("NIMBUS_PRINT_URL = '%s'" % printurl) + lines.append("NIMBUS_TOKEN_EXPIRE_HOURS = %d" % expire_hours) + + generated_text = "\n" + for line in lines: + generated_text += line + generated_text += "\n" + + log.debug("Going to write this to generated_settings:\n%s" % generated_text) + + f = open(generated_settings, 'w') + f.write(generated_text) + f.close() + pathutil.ensure_file_exists(generated_settings, "generated web settings") + print "Wrote generated_settings: %s" % generated_settings + + # -------------------------------------------------------------------------- + + generated_secrets = pathutil.pathjoin(basedir, "src/python/nimbusweb/portal/generated_secrets.py") + if not pathutil.check_path_exists(generated_secrets): + + # Creating secret each newconf would mean that people's sessions won't + # work after webapp reboot and they would need to login again. + # Instead, it is only written when nonexistent (clean-slate script will + # remove it). + lines = [] + okchars = string.letters + string.digits + "!@%^_&*+-" + okchars += okchars + secret = ''.join( Random().sample(okchars, 50) ) + lines.append("SECRET_KEY = '%s'" % secret) + + generated_text = "\n" + for line in lines: + generated_text += line + generated_text += "\n" + + f = open(generated_secrets, 'w') + f.write(generated_text) + f.close() + pathutil.ensure_file_exists(generated_secrets, "generated web secrets") + print "Wrote generated_secrets: %s" % generated_secrets diff --git a/ctx-broker/home/lib/run-broker.sh b/ctx-broker/home/lib/run-broker.sh new file mode 100644 index 00000000..ba30cec5 --- /dev/null +++ b/ctx-broker/home/lib/run-broker.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +NIMBUS_HOME_REL="`dirname $0`/.." +NIMBUS_HOME=`cd $NIMBUS_HOME_REL; pwd` + +. "$NIMBUS_HOME/var/environment.sh" + +LOGFILE="$NIMBUS_HOME/var/broker.log" +PORT="8443" + +# Changing CWD is necessary only to keep relative path settings for the +# "main.xml" spring paths in "services/etc/nimbus/jndi-config.xml" +# Everything else is run through a $NIMBUS_HOME translator. + +cd $GLOBUS_LOCATION + +exec $NIMBUS_HOME/services/bin/globus-start-container -p $PORT >> $LOGFILE 2>&1 diff --git a/ctx-broker/home/lib/run-pynimbusconfig-tests.sh b/ctx-broker/home/lib/run-pynimbusconfig-tests.sh new file mode 100755 index 00000000..ef69f5c2 --- /dev/null +++ b/ctx-broker/home/lib/run-pynimbusconfig-tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Allow this to run from either the source tree or a NIMBUS_HOME deployment +NIMBUS_HOMEDIR_REL="`dirname $0`/../../.." +NIMBUS_HOMEDIR=`cd $NIMBUS_HOMEDIR_REL; pwd` +if [ -d $NIMBUS_HOMEDIR/ctx-broker/home/lib/pynimbusconfig ]; then + NIMBUS_PYLIB=$NIMBUS_HOMEDIR/ctx-broker/home/lib +else + NIMBUS_HOMEDIR_REL="`dirname $0`/.." + NIMBUS_HOMEDIR=`cd $NIMBUS_HOMEDIR_REL; pwd` + if [ -d $NIMBUS_HOMEDIR/lib/pynimbusconfig ]; then + NIMBUS_PYLIB=$NIMBUS_HOMEDIR/lib + else + echo "Cannot locate Python lib directory" + exit 1 + fi +fi + +PYTHONPATH="$NIMBUS_PYLIB:$PYTHONPATH" +export PYTHONPATH + +nosetests --nologcapture -x -v -w $NIMBUS_PYLIB/pynimbusconfig From 18bd32e96a85db131f93c863f950fb09e5ceba09 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Fri, 10 Jun 2011 14:14:37 -0500 Subject: [PATCH 07/21] Separate build & dist system for standalone context broker, some ant-code dupe --- scripts/broker-make-dist.sh | 54 ++++ scripts/gt/broker-build-and-install.sh | 2 +- scripts/install-broker-only | 230 ++++++++++++++++++ scripts/lib/gt4.0/brokerdist/build.properties | 32 +++ scripts/lib/gt4.0/brokerdist/build.xml | 160 ++++++++++++ .../lib/gt4.0/brokerdist/topdocs/LICENSE.txt | 193 +++++++++++++++ .../lib/gt4.0/brokerdist/topdocs/README.txt | 15 ++ scripts/lib/gt4.0/build/build.properties | 2 +- scripts/lib/gt4.0/build/build.xml | 37 ++- scripts/lib/gt4.0/dist/build.properties | 4 +- scripts/lib/gt4.0/dist/build.xml | 13 +- scripts/make-dist.sh | 1 + 12 files changed, 708 insertions(+), 35 deletions(-) create mode 100755 scripts/broker-make-dist.sh create mode 100755 scripts/install-broker-only create mode 100644 scripts/lib/gt4.0/brokerdist/build.properties create mode 100644 scripts/lib/gt4.0/brokerdist/build.xml create mode 100644 scripts/lib/gt4.0/brokerdist/topdocs/LICENSE.txt create mode 100644 scripts/lib/gt4.0/brokerdist/topdocs/README.txt diff --git a/scripts/broker-make-dist.sh b/scripts/broker-make-dist.sh new file mode 100755 index 00000000..fedf84b0 --- /dev/null +++ b/scripts/broker-make-dist.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +if ([ "X$1" == "X--help" ] || [ "X$1" == "X-h" ]); then + echo "Usage:" + echo "broker-make-dist.sh [] []" + echo "Default: make-dist . git://github.com/nimbusproject/nimbus.git" + exit 0 +fi + +if [ "X$2" != "X" ]; then +# normalize dir, otherwise it is relative to $co_dir + dest_dir=`cd "$1";pwd` +else + dest_dir=`pwd` +fi + +if [ "X$2" != "X" ]; then + repo=$2 +else + repo="git://github.com/nimbusproject/nimbus.git" +fi + +co_dir=`mktemp -d -t tmp.XXXXXXXXXX` + +echo "Destination directory: $dest_dir" +echo "Nimbus git repository: $repo" +echo "Created temp directory: $co_dir" + +cd $co_dir +git clone $repo + +cd $co_dir/nimbus +ant -f scripts/lib/gt4.0/dist/build.xml clean +ant -f scripts/lib/gt4.0/brokerdist/build.xml clean-local +ant -f scripts/lib/gt4.0/brokerdist/build.xml create-dist +if [ $? -ne 0 ]; then + echo "create-dist failed" + exit 1 +fi + +ls scripts/lib/gt4.0/brokerdist/result/ + +cd scripts/lib/gt4.0/brokerdist/result/ + +cp *.tar.gz $dest_dir +if [ $? -ne 0 ]; then + echo "copy failed" + exit 1 +fi + +echo "Removing temp dir: $co_dir" +rm -rf $co_dir + +echo "Finished. Tarballs copied to $dest_dir." diff --git a/scripts/gt/broker-build-and-install.sh b/scripts/gt/broker-build-and-install.sh index 9ee736fe..e39666da 100755 --- a/scripts/gt/broker-build-and-install.sh +++ b/scripts/gt/broker-build-and-install.sh @@ -1,6 +1,6 @@ #!/bin/bash -NIMBUS_PRINTNAME="build and install Context Broker" +NIMBUS_PRINTNAME="Build and install Context Broker" NIMBUS_ANT_CMD="deploy-broker -Dbuild.also=x $*" BASEDIR_REL="`dirname $0`/../.." diff --git a/scripts/install-broker-only b/scripts/install-broker-only new file mode 100755 index 00000000..406dd082 --- /dev/null +++ b/scripts/install-broker-only @@ -0,0 +1,230 @@ +#!/bin/bash + +GUIDEURL="http://www.nimbusproject.org/docs/2.8/" + +FORCE_FRESH_INSTALL="yes" + +NIMBUS_SRC_REL="`dirname $0`" +NIMBUS_SRC=`cd $NIMBUS_SRC_REL; pwd` + +if [ "X$1" = "X" ]; then + echo "" + echo "Usage: $0 destination_dir" + echo " You must specify the destination directory." + echo "" + exit 1 +fi + +NIMBUS_REL_BASE_HOME="`dirname $1`" +NIMBUS_BASE_HOME=`cd $NIMBUS_REL_BASE_HOME; pwd` + +if [ ! -e $NIMBUS_REL_BASE_HOME ]; then + echo "The parent directory for the installation path $1 must exist" + exit 1 +fi + +NIMBUS_INST_DIR=`basename $1` +NIMBUS_HOME="$NIMBUS_BASE_HOME/$NIMBUS_INST_DIR" +echo "NIMBUS_HOME $NIMBUS_HOME" + +if [ "X$NIMBUS_ALLOW_ROOT_INSTALL" != "Xyes" ] && [ $EUID -eq 0 ]; then + echo "" + echo "It is not recommended to install or run the Context Broker as root." + echo "If you insist, set the NIMBUS_ALLOW_ROOT_INSTALL environment variable to \"yes\"" + echo "" + exit 1 +fi + +if [ -d $NIMBUS_HOME ] && [ "$(ls -A $NIMBUS_HOME)" ]; then + if [ $FORCE_FRESH_INSTALL = "yes" ]; then + echo "" + echo "The destination directory '$NIMBUS_HOME' exists and is not empty." + echo "It is not recommended to reinstall the Context Broker into an existing install." + echo "" + echo "If you know what you are doing and want to reinstall, edit this script:" + echo " $0" + echo "and change FORCE_FRESH_INSTALL to \"no\"" + echo "" + echo "It is highly unlikely you want this. Note you may not install" + echo "the context broker to the the same directory as a Nimbus IaaS" + echo "installation." + echo "" + + exit 1 + fi +fi + +if [ ! -d $NIMBUS_HOME ]; then + mkdir $NIMBUS_HOME +fi + +echo "Installing from $NIMBUS_SRC" + +CONTAINER_URL="https://github.com/downloads/nimbusproject/nimbus/ws-core-4.0.8-bin.tar.gz" +CONTAINER_TARNAME="ws-core-4.0.8-bin.tar.gz" +CONTAINER_UNTARREDNAME="ws-core-4.0.8" + +# destination directory inside $NIMBUS_HOME +CONTAINER_DIRNAME="services" + + +TMPDIR="$NIMBUS_SRC/tmp" + +if [ ! -d $TMPDIR ]; then + mkdir $TMPDIR + if [ $? -ne 0 ]; then + echo "Failed to create temp directory: $TMPDIR" + exit 1 + fi +fi + +echo "" +echo "-----------------------------------------------------------------" +echo " Preparing Context Broker installation directory" +echo "-----------------------------------------------------------------" +echo "" +# first ensure we have a GT tarball. Most likely it has been bundled with +# the Context Broker installer. +if [ ! -f $TMPDIR/$CONTAINER_TARNAME ]; then + echo "Downloading service container.." + echo "" + wget --no-check-certificate -c -O $TMPDIR/$CONTAINER_TARNAME $CONTAINER_URL + + if [ $? -ne 0 ]; then + echo "" + echo "Failed to download container tarball from: $CONTAINER_URL" + echo "" + echo "If you are unable to access the Internet, please retrieve this" + echo "file by other means and place it in $TMPDIR/." + echo "Then rerun this script." + + rm -f $TMPDIR/$CONTAINER_TARNAME + + exit 1 + fi +fi + + +CONTAINER_DIR="$NIMBUS_HOME/$CONTAINER_DIRNAME" +if [ ! -d $CONTAINER_DIR ]; then + + echo "Installing container.." + + tar xzf $TMPDIR/$CONTAINER_TARNAME -C $TMPDIR + if [ $? -ne 0 ]; then + echo "Failed to expand container tarball: $TMPDIR/$CONTAINER_TARNAME" + echo "If this file is corrupted, you may delete it and rerun this script to download a new one." + exit 1 + fi + + mv $TMPDIR/$CONTAINER_UNTARREDNAME $CONTAINER_DIR + if [ $? -ne 0 ]; then + echo "Failed to move container directory to $CONTAINER_DIR" + exit 1 + fi + + CONTAINER_LOG_PROPS="$CONTAINER_DIR/container-log4j.properties" + echo "" >> $CONTAINER_LOG_PROPS + echo "# Change the following line from \"INFO\" to \"DEBUG\" to enable debugging" >> $CONTAINER_LOG_PROPS + echo "# Logging changes require a container restart." >> $CONTAINER_LOG_PROPS + echo "log4j.category.org.nimbustools=INFO" >> $CONTAINER_LOG_PROPS + +else + echo "Service container already exists at $CONTAINER_DIR" +fi + +echo "" +echo "-----------------------------------------------------------------" +echo " Building and installing Nimbus Context Broker" +echo "-----------------------------------------------------------------" + +GLOBUS_LOCATION=$CONTAINER_DIR +export GLOBUS_LOCATION + +$NIMBUS_SRC/scripts/gt/broker-build-and-install.sh +if [ $? -ne 0 ]; then + echo "Build and install FAILED!" + exit 1 +fi + +echo "Deploying skeleton directory structure.." +cp -fr $NIMBUS_SRC/home/* $NIMBUS_HOME/ +if [ $? -ne 0 ]; then + echo "Failed to copy Context Broker home directory" + exit 1 +fi + +echo "" +echo "-----------------------------------------------------------------" +echo " Configuring installed services" +echo "-----------------------------------------------------------------" + +# Override the default nimbus-setup.conf + +if [ ! -e $NIMBUS_HOME/nimbus-setup.conf ]; then + + cat >> $NIMBUS_HOME/nimbus-setup.conf << "EOF" +[nimbussetup] + +# relative to base directory +hostcert: var/hostcert.pem +hostkey: var/hostkey.pem +ca.dir: var/ca +ca.trustedcerts.dir: var/ca/trusted-certs + +gridmap: services/etc/nimbus-context-broker/user-mapfile +envfile: var/environment.sh + +keystore: var/keystore.jks +keystore.pass: changeit + +broker.enabled: True +services.wait: 10 +EOF + +fi + +if [ ! -e $NIMBUS_HOME/var ]; then + mkdir $NIMBUS_HOME/var + if [ $? -ne 0 ]; then + echo "Could not make the var directory: $NIMBUS_HOME/var" + exit 1 + fi + chmod +x $NIMBUS_HOME/bin/* + if [ $? -ne 0 ]; then + echo "Could not prepare the bin directory: $NIMBUS_HOME/bin" + exit 1 + fi +fi + +CONFIG_SCRIPT="$NIMBUS_HOME/bin/broker-configure" + +if [ ! -f $CONFIG_SCRIPT ]; then + echo "Configuration script could not be found: $CONFIG_SCRIPT" + exit 1 +fi + +$CONFIG_SCRIPT $EXTRA_CONFIG_SCRIPT_ARGS +if [ $? -ne 0 ]; then + echo "Broker configuration script failed! You may try running it manually:" + echo " $CONFIG_SCRIPT" + echo "You can also run the script with debugging output:" + echo " $CONFIG_SCRIPT --debug" + exit 1 +fi + +echo "" +echo "-----------------------------------------------------------------" +echo " Context Broker installation succeeded!" +echo "-----------------------------------------------------------------" +echo "" +echo "Additional configuration may be necessary, refer to this URL for information:" +echo "" +echo " $GUIDEURL" +echo "" +echo "You can start/stop the Context Broker with the brokerctl command. e.g:" +echo "" +echo " $NIMBUS_HOME/bin/brokerctl start" +echo "" + +exit 0 diff --git a/scripts/lib/gt4.0/brokerdist/build.properties b/scripts/lib/gt4.0/brokerdist/build.properties new file mode 100644 index 00000000..9d3f5a78 --- /dev/null +++ b/scripts/lib/gt4.0/brokerdist/build.properties @@ -0,0 +1,32 @@ +# Nimbus distribution properties + +ctxbroker.version=2.8RC1 + +binary.suffix=bin +source.suffix=src +tgz.suffix=tar.gz + +dist.dir=result +distbuild.dir=build + +# ============================================================ + +base.name=nimbus-ctxbroker- +nimbus.main=${base.name}${ctxbroker.version}-${source.suffix} + +# ============================================================ + +nimbus.main.builddir=../build + +nimbus.basedir=../../../../ + +nimbus.web.autocommon.distbuild.dir=autocommonbuild + +# ============================================================ + +wscore.url=http://www-unix.globus.org/ftppub/gt4/4.0/4.0.8/ws-core/bin/ws-core-4.0.8-bin.tar.gz +wscore.destdir=${nimbus.basedir}/tmp/ +wscore.dest=${wscore.destdir}/ws-core-4.0.8-bin.tar.gz + +# you can override this in ~/nimbus.build.properties +nimbus.java.compilerarg=-nowarn diff --git a/scripts/lib/gt4.0/brokerdist/build.xml b/scripts/lib/gt4.0/brokerdist/build.xml new file mode 100644 index 00000000..40502e99 --- /dev/null +++ b/scripts/lib/gt4.0/brokerdist/build.xml @@ -0,0 +1,160 @@ + + + + + + Nimbus Ctx Broker buildfile to generate releases. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/lib/gt4.0/brokerdist/topdocs/LICENSE.txt b/scripts/lib/gt4.0/brokerdist/topdocs/LICENSE.txt new file mode 100644 index 00000000..e5407660 --- /dev/null +++ b/scripts/lib/gt4.0/brokerdist/topdocs/LICENSE.txt @@ -0,0 +1,193 @@ + +Copyright 1999-2010 University of Chicago + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy +of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +-------------------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/scripts/lib/gt4.0/brokerdist/topdocs/README.txt b/scripts/lib/gt4.0/brokerdist/topdocs/README.txt new file mode 100644 index 00000000..db65ecb0 --- /dev/null +++ b/scripts/lib/gt4.0/brokerdist/topdocs/README.txt @@ -0,0 +1,15 @@ +_________________________________________________________________________ +Nimbus http://www.nimbusproject.org + + Open Source Nimbus Context Broker +_________________________________________________________________________ + +Documentation is available at: + http://www.nimbusproject.org/docs/ + +To install, run: + + ./install /path/to/destination + +Subscribe to the mailing list for support: + http://lists.globus.org/mailman/listinfo/workspace-user diff --git a/scripts/lib/gt4.0/build/build.properties b/scripts/lib/gt4.0/build/build.properties index 8e6c29b7..bd8664ab 100644 --- a/scripts/lib/gt4.0/build/build.properties +++ b/scripts/lib/gt4.0/build/build.properties @@ -39,8 +39,8 @@ nimbus.autocommon.gt4_0.builddir=${nimbus.basedir}/autocommon/ # Context broker makes its own GAR nimbus.ctx-broker.gt4_0.builddir=${nimbus.basedir}/ctx-broker/ -# GT4.0.x wizard implementation has dep on autocommon nimbus.authzdb.builddir=${nimbus.basedir}/authzdb +nimbus.querygeneral.builddir=${nimbus.basedir}/query # you can override this in ~/nimbus.build.properties nimbus.java.compilerarg=-nowarn diff --git a/scripts/lib/gt4.0/build/build.xml b/scripts/lib/gt4.0/build/build.xml index 2037b4df..f83ee494 100644 --- a/scripts/lib/gt4.0/build/build.xml +++ b/scripts/lib/gt4.0/build/build.xml @@ -131,6 +131,7 @@ + @@ -145,12 +146,21 @@ - + + + + + + + + + + @@ -238,35 +247,17 @@ - - - - - - - - - Building Context Broker: - - - - - + depends="checkgl, build-for-standalone-broker"> - Default Context Broker installation to GT4.0 GLOBUS_LOCATION: + Standalone Context Broker installation: - + + @@ -339,8 +348,8 @@ - - + + @@ -502,8 +511,16 @@ - - + + + + @@ -597,7 +614,7 @@ - + Date: Thu, 23 Jun 2011 09:57:16 -0700 Subject: [PATCH 11/21] Update persistence schema for failuresuites --- .../lib/workspace_service_derby_schema.sql | 95 ++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/service/service/java/tests/suites/failure/home/services/share/nimbus/lib/workspace_service_derby_schema.sql b/service/service/java/tests/suites/failure/home/services/share/nimbus/lib/workspace_service_derby_schema.sql index 1ebe9f52..485707c8 100644 --- a/service/service/java/tests/suites/failure/home/services/share/nimbus/lib/workspace_service_derby_schema.sql +++ b/service/service/java/tests/suites/failure/home/services/share/nimbus/lib/workspace_service_derby_schema.sql @@ -264,12 +264,101 @@ creator_is_superuser SMALLINT, group_id VARCHAR(512), ssh_key_name VARCHAR(512), creation_time BIGINT, -nics VARCHAR(512) +nics VARCHAR(512), +status VARCHAR(512) ); -- Persistence for AsyncRequest list of NICs +--CREATE TABLE async_requests_vms +--( +--id VARCHAR(512), +--vmid INT +--); + +-- +-- Persistence for async virtual machines: + CREATE TABLE async_requests_vms ( -id VARCHAR(512), -vmid INT +async_id VARCHAR(512) NOT NULL, +binding_index INT NOT NULL, +id INT NOT NULL, +name VARCHAR(128) NOT NULL, +node VARCHAR(128), +prop_required SMALLINT NOT NULL, +unprop_required SMALLINT NOT NULL, +network VARCHAR(1024), +kernel_parameters VARCHAR(128), +vmm VARCHAR(32), +vmm_version VARCHAR(32), +assocs_needed VARCHAR(256), +md_user_data VARCHAR(30720), +preemptable SMALLINT, +credential_name VARCHAR(128), +PRIMARY KEY (async_id, binding_index, id) +); + +-- +-- Persistence for async vm deployment-time data: + +CREATE TABLE async_requests_vm_deployment +( +async_id VARCHAR(512) NOT NULL, +binding_index INT NOT NULL, +vmid INT NOT NULL, +requested_state SMALLINT, +requested_shutdown SMALLINT, +min_duration INT, +ind_physmem INT, +ind_physcpu INT +); + +-- +-- async VM partitions + +CREATE TABLE async_requests_vm_partitions +( +async_id VARCHAR(512) NOT NULL, +binding_index INT NOT NULL, +vmid INT NOT NULL, +image VARCHAR(4096) NOT NULL, +imagemount VARCHAR(128) NOT NULL, +readwrite SMALLINT NOT NULL, +rootdisk SMALLINT NOT NULL, +blankspace INT NOT NULL, +prop_required SMALLINT NOT NULL, +unprop_required SMALLINT NOT NULL, +alternate_unprop VARCHAR(128) ); + +-- +-- Persistence for async vm file copy tasks + +CREATE TABLE async_requests_vm_file_copy +( +async_id VARCHAR(512) NOT NULL, +binding_index INT NOT NULL, +vmid INT NOT NULL, +sourcepath VARCHAR(36) NOT NULL, +destpath VARCHAR(512), +on_image SMALLINT NOT NULL +); + +CREATE TABLE async_requests_allocated_vms +( +id VARCHAR(512) NOT NULL, +vmid INT NOT NULL +); + +CREATE TABLE async_requests_finished_vms +( +id VARCHAR(512) NOT NULL, +vmid INT NOT NULL +); + +CREATE TABLE async_requests_to_be_preempted +( +id VARCHAR(512) NOT NULL, +vmid INT NOT NULL +); + From e7c763d806b94c51505934d00c0934576fae2522 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 13:39:49 -0500 Subject: [PATCH 12/21] enable the new test suite in Jenkins --- scripts/integration-suites.sh | 2 +- service/service/java/tests/suites/build.xml | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/integration-suites.sh b/scripts/integration-suites.sh index fe9aa515..a5da9a73 100755 --- a/scripts/integration-suites.sh +++ b/scripts/integration-suites.sh @@ -1,7 +1,7 @@ #!/bin/bash -ALL_TEST_SUITES="basic01 basic02 basic03 basic04 basic05 basic06 failure01 failure02 spot01 spot02 spot03 spot04 spot05 spot06" +ALL_TEST_SUITES="basic01 basic02 basic03 basic04 basic05 basic06 basic07 failure01 failure02 spot01 spot02 spot03 spot04 spot05 spot06" if [ "X" == "X$1" ]; then diff --git a/service/service/java/tests/suites/build.xml b/service/service/java/tests/suites/build.xml index f6520bab..e18b5c99 100644 --- a/service/service/java/tests/suites/build.xml +++ b/service/service/java/tests/suites/build.xml @@ -204,7 +204,13 @@ - + + + + + + + From 100f5031881a8107e724c1d89186385ba30d99a0 Mon Sep 17 00:00:00 2001 From: Patrick Armstrong Date: Thu, 23 Jun 2011 12:06:05 -0700 Subject: [PATCH 13/21] Revert back to correct spring config files for spot and failure tests --- .../nimbus/workspace-service/other/main.xml | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml b/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml index 8b61c7b3..44700f3a 100644 --- a/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml +++ b/service/service/java/tests/suites/failure/home/services/etc/nimbus/workspace-service/other/main.xml @@ -97,16 +97,7 @@ - - - + @@ -348,8 +339,8 @@ - - + + @@ -511,16 +502,8 @@ - - - - + + From 608ce2499465a7b06b7f63fc1be66845f4444085 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 15:43:41 -0500 Subject: [PATCH 14/21] small rework of changelog --- docs/src/changelog.html | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/src/changelog.html b/docs/src/changelog.html index c6f7ab7d..7abbe2a2 100644 --- a/docs/src/changelog.html +++ b/docs/src/changelog.html @@ -22,17 +22,24 @@

Changelog

  • - This release contains many important bug fixes as well as a few new features. + This release contains many important bug fixes as well as some new features.

  • -Image propagation performance enhancing features have been introduced in. -Propagation by means of a file system copy has been introduced. This -can greatly decrease the boot time of VMs on systems where a fast shared -file system is used. We have also added support for a VM image cache on -all VMMs. This greatly increases the performance for clouds with a -base image that is booted often (a common use case). +Propagation by means of a file system copy: this can greatly decrease the boot time of VMs on systems where a fast shared file system exists. +

    +
    • +
  • +

    +VM image caching: this will greatly increase the boot performance for clouds with a base image that is launched often (a common use case). This works with any propagation method. +

    +
    • +
  • +

    + libvirt template support added. A cloud administrator can + now completely control the options sent to libvirt when starting + a virtual machine by editing the template file

@@ -165,7 +172,8 @@

Changelog

A cache of propagated images can now be kept on each VMM. Before an image is propagated the cache is checked for an image with a matching checksum. If found that image is used and no propagation - is needed which can save a significant amount of time. + is needed which can save a significant amount of time. This will + work with any propagation mechanism.

  • From 5b350d0e385b08e18a3ea8e20ef55306f727d9a8 Mon Sep 17 00:00:00 2001 From: David LaBissoniere Date: Thu, 23 Jun 2011 15:50:59 -0500 Subject: [PATCH 15/21] Include units in cache max size config comment --- control/etc/workspace-control/images.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/control/etc/workspace-control/images.conf b/control/etc/workspace-control/images.conf index 42971d3f..3947615c 100644 --- a/control/etc/workspace-control/images.conf +++ b/control/etc/workspace-control/images.conf @@ -42,7 +42,8 @@ blankcreate: blankcreate.sh # if the image is already local. # - this directory has the same constraints as securelocaldir # - this directory can not be a path on a network file system -# - the max size of the cache can be set, 0 (default) sets it to unlimited +# - the max size (in bytes) of the cache can be set, 0 (default) sets it +# to unlimited #cachedir: imagecache #cache_size: 0 From 5bb03ee1a8a72835757e6950abf3a1c37d8553d3 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 16:23:24 -0500 Subject: [PATCH 16/21] make the test wait longer for termination backoff to kick in --- .../globus/workspace/testing/suites/failure/TerminateSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/TerminateSuite.java b/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/TerminateSuite.java index 69696c38..3bca64a5 100644 --- a/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/TerminateSuite.java +++ b/service/service/java/tests/suites/failure/src/org/globus/workspace/testing/suites/failure/TerminateSuite.java @@ -530,7 +530,7 @@ public void unpropOneFail() throws Exception { MockShutdownTrash.setFail(false); logger.warn("Set to succeed."); - Thread.sleep(8000L); + Thread.sleep(10000L); // Should now be gone: assertFalse(rm.exists(vms[0].getID(), Manager.INSTANCE)); From 7135a05b405a61238c5ce84ffd7cabbf342beef1 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 17:15:55 -0500 Subject: [PATCH 17/21] highlighted ImportKeyPair impl --- docs/src/changelog.html | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/src/changelog.html b/docs/src/changelog.html index 7abbe2a2..1a1937f8 100644 --- a/docs/src/changelog.html +++ b/docs/src/changelog.html @@ -42,6 +42,11 @@

    Changelog

    a virtual machine by editing the template file

    • +
  • +

    + ImportKeyPair is implemented in the EC2 protocols (details below). +

    +
    • 2.8 - IaaS Services @@ -61,6 +66,14 @@

    Changelog

    in addition to being part of the default IaaS installation.

    +
  • +

    + Implemented EC2 ImportKeyPair operation. The old CreateKeyPair behavior is disabled (the behavior of allowing a "||" token based import mechanism with CreateKeyPair). +

    +

    + EC2 used to only have CreateKeyPair, now the EC2 protocols in Nimbus IaaS use ImportKeyPair and CreateKeyPair as intended as operations. The old behavior can be enabled if the administrator wants. (enhancement 36). +

    +

  • Documentation added to the zero to cloud guide describing @@ -108,11 +121,6 @@

    Changelog

    Needed networks not being respected

    • -
  • -

    - Implement EC2 ImportKeyPair -

    -
    • From 65f351f4d7e2c6cbdd89ebd717f084ff6094dfe1 Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 19:01:15 -0500 Subject: [PATCH 18/21] Needed to adjust service suites for new jar directory layout --- .idea-modules/service-suites/tests-common.iml | 1 + service/service/java/tests/suites/build.properties | 1 + service/service/java/tests/suites/build.xml | 3 +++ .../src/org/globus/workspace/testing/NimbusTestBase.java | 6 +++--- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.idea-modules/service-suites/tests-common.iml b/.idea-modules/service-suites/tests-common.iml index 42826f29..2ee69a5c 100644 --- a/.idea-modules/service-suites/tests-common.iml +++ b/.idea-modules/service-suites/tests-common.iml @@ -11,6 +11,7 @@ + diff --git a/service/service/java/tests/suites/build.properties b/service/service/java/tests/suites/build.properties index 42eb337e..17f67862 100644 --- a/service/service/java/tests/suites/build.properties +++ b/service/service/java/tests/suites/build.properties @@ -5,6 +5,7 @@ nimbus.st.service.main.builddir=${nimbus.st.basedir}/service/service/java/source nimbus.st.metadata.builddir=${nimbus.st.basedir}/metadata/java/source nimbus.st.authzdb.builddir=${nimbus.st.basedir}/authzdb nimbus.st.main.lib.dir=${nimbus.st.basedir}/lib/services +nimbus.st.workspaceservice.lib.dir=${nimbus.st.basedir}/lib/workspaceservice nimbus.st.test.lib.dir=${nimbus.st.basedir}/lib/test nimbus.st.service.api.dist.dir=${nimbus.st.basedir}/service-api/java/source/dist nimbus.st.service.main.dist.dir=${nimbus.st.basedir}/service/service/java/source/dist diff --git a/service/service/java/tests/suites/build.xml b/service/service/java/tests/suites/build.xml index e18b5c99..953793e9 100644 --- a/service/service/java/tests/suites/build.xml +++ b/service/service/java/tests/suites/build.xml @@ -17,6 +17,9 @@ + + + diff --git a/service/service/java/tests/suites/common/src/org/globus/workspace/testing/NimbusTestBase.java b/service/service/java/tests/suites/common/src/org/globus/workspace/testing/NimbusTestBase.java index cf80d706..052654e9 100644 --- a/service/service/java/tests/suites/common/src/org/globus/workspace/testing/NimbusTestBase.java +++ b/service/service/java/tests/suites/common/src/org/globus/workspace/testing/NimbusTestBase.java @@ -466,19 +466,19 @@ protected void setUpShareDir(String nimbusHome) throws Exception { String libdir = null; String apath = nh.getAbsolutePath(); while (apath != null) { - final File candidate = candidate(apath, "lib/services/"); + final File candidate = candidate(apath, "lib/workspaceservice/"); if (candidate != null) { libdir = candidate.getAbsolutePath(); } apath = new File(apath).getParent(); } if (libdir == null) { - throw new Exception("could not determine proper lib/services directory, " + + throw new Exception("could not determine proper lib/workspaceservice directory, " + "is your nimbus home somewhere outside of the repository"); } final File sanityCheck = new File(libdir, "derby.jar"); if (!sanityCheck.exists()) { - throw new Exception("could not determine proper lib/services directory, " + + throw new Exception("could not determine proper lib/workspaceservice directory, " + "is your nimbus home somewhere outside of the repository (derby.jar not found)"); } conf.setProperty("derby.classpath.dir.prop", libdir); From 31f24aae803609e57711ffabb537775045a063cf Mon Sep 17 00:00:00 2001 From: Tim Freeman Date: Thu, 23 Jun 2011 19:25:52 -0500 Subject: [PATCH 19/21] recent addition of DB to msgbridge needs these jars --- messaging/gt4.0-elastic/java/msgbridge/build.properties | 1 + messaging/gt4.0-elastic/java/msgbridge/build.xml | 4 ++++ messaging/gt4.0/java/common/build.properties | 1 + messaging/gt4.0/java/common/build.xml | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/messaging/gt4.0-elastic/java/msgbridge/build.properties b/messaging/gt4.0-elastic/java/msgbridge/build.properties index d79ad079..489b1c7e 100644 --- a/messaging/gt4.0-elastic/java/msgbridge/build.properties +++ b/messaging/gt4.0-elastic/java/msgbridge/build.properties @@ -7,6 +7,7 @@ nimbus.messaging.gt4_0-elastic.dist.dir=dist nimbus.messaging.gt4_0-elastic.jar.name=nimbus-messaging-elastic-bridge-gt4.0.jar nimbus.messaging.gt4_0-elastic.lib.dir=../../../../lib/services +nimbus.messaging.gt4_0-elastic.workspaceservice.lib.dir=../../../../lib/workspaceservice # Dependencies from source tree nimbus.messaging.gt4_0.common.dist.dir=../../../gt4.0/java/common/dist/ diff --git a/messaging/gt4.0-elastic/java/msgbridge/build.xml b/messaging/gt4.0-elastic/java/msgbridge/build.xml index 82eb4484..75bd760f 100644 --- a/messaging/gt4.0-elastic/java/msgbridge/build.xml +++ b/messaging/gt4.0-elastic/java/msgbridge/build.xml @@ -62,6 +62,10 @@     + + + + diff --git a/messaging/gt4.0/java/common/build.properties b/messaging/gt4.0/java/common/build.properties index 1246aaee..a83611d3 100644 --- a/messaging/gt4.0/java/common/build.properties +++ b/messaging/gt4.0/java/common/build.properties @@ -7,6 +7,7 @@ nimbus.messaging.gt4_0.common.dist.dir=dist nimbus.messaging.gt4_0.common.jar.name=nimbus-messaging-common-gt4.0.jar nimbus.messaging.gt4_0.common.lib.dir=../../../../lib/services +nimbus.messaging.gt4_0.workspaceservice.lib.dir=../../../../lib/workspaceservice # Dependencies from source tree nimbus.service.api.dist.dir=../../../../service-api/java/source/dist/ diff --git a/messaging/gt4.0/java/common/build.xml b/messaging/gt4.0/java/common/build.xml index 46549c08..a59f181c 100644 --- a/messaging/gt4.0/java/common/build.xml +++ b/messaging/gt4.0/java/common/build.xml @@ -57,6 +57,10 @@     + + + + From b93f6c5add0b9ba8fecd85017668148e7aa883f5 Mon Sep 17 00:00:00 2001 From: David LaBissoniere Date: Fri, 24 Jun 2011 11:32:34 -0500 Subject: [PATCH 20/21] Swapped group01 and group04 authz properties group01 is the default and it should be unlimited. --- .../group-authz/group01.properties | 41 +++++++------------ .../group-authz/group04.properties | 41 ++++++++++++------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/service/service/java/source/etc/workspace-service/group-authz/group01.properties b/service/service/java/source/etc/workspace-service/group-authz/group01.properties index 9b5c7904..18c04483 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group01.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group01.properties @@ -1,33 +1,22 @@ -# For logging and other side-purposes -vws.group.authz.groupname=TESTING +vws.group.authz.groupname=UNLIMITED -# Maximum currently reserved minutes at one point in time -# If the caller has two other workspaces with 600 minutes scheduled -# for each, the value being checked against this policy would be 1200 minutes -# plus whatever time the current request is for. -vws.group.authz.maxReservedMinutes=300 +# 0 means no restriction +vws.group.authz.maxReservedMinutes=0 -# Maximum elapsed (finished) and currently reserved minutes at one point in time -# If the caller has one other workspace with 600 minutes scheduled and 6000 -# minutes of recorded past usage, the value being checked against this policy -# would be 6600 minutes plus whatever time the current request is for. This is -# the all-time maximum usage cap. -vws.group.authz.maxElapsedReservedMinutes=300 +# 0 means no restriction +vws.group.authz.maxElapsedReservedMinutes=0 -# Maximum number of CPUs per VM instance -vws.group.authz.maxCPUs=2 +# 0 means no restriction +vws.group.authz.maxWorkspaceNumber=0 -# Maximum number of running workspaces at one point in time -vws.group.authz.maxWorkspaceNumber=5 +# 0 means no restriction +vws.group.authz.maxWorkspacesInGroup=0 -# Maximum number of workspaces per request -vws.group.authz.maxWorkspacesInGroup=1 +# not present means no restriction +#vws.group.authz.imageNodeHostname= -# Image node -vws.group.authz.imageNodeHostname=example.com +# not present means no restriction +#vws.group.authz.imageBaseDirectory= -# Image node basedir -vws.group.authz.imageBaseDirectory=/cloud - -# Use DN hashing convention for user-specific subdirectories (see documentation) -vws.group.authz.dirHashMode=true +# not present means false +#vws.group.authz.dirHashMode= diff --git a/service/service/java/source/etc/workspace-service/group-authz/group04.properties b/service/service/java/source/etc/workspace-service/group-authz/group04.properties index 61f2eb82..9b5c7904 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group04.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group04.properties @@ -1,22 +1,33 @@ -vws.group.authz.groupname=SUPERUSER +# For logging and other side-purposes +vws.group.authz.groupname=TESTING -# 0 means no restriction -vws.group.authz.maxReservedMinutes=0 +# Maximum currently reserved minutes at one point in time +# If the caller has two other workspaces with 600 minutes scheduled +# for each, the value being checked against this policy would be 1200 minutes +# plus whatever time the current request is for. +vws.group.authz.maxReservedMinutes=300 -# 0 means no restriction -vws.group.authz.maxElapsedReservedMinutes=0 +# Maximum elapsed (finished) and currently reserved minutes at one point in time +# If the caller has one other workspace with 600 minutes scheduled and 6000 +# minutes of recorded past usage, the value being checked against this policy +# would be 6600 minutes plus whatever time the current request is for. This is +# the all-time maximum usage cap. +vws.group.authz.maxElapsedReservedMinutes=300 -# 0 means no restriction -vws.group.authz.maxWorkspaceNumber=0 +# Maximum number of CPUs per VM instance +vws.group.authz.maxCPUs=2 -# 0 means no restriction -vws.group.authz.maxWorkspacesInGroup=0 +# Maximum number of running workspaces at one point in time +vws.group.authz.maxWorkspaceNumber=5 -# not present means no restriction -#vws.group.authz.imageNodeHostname= +# Maximum number of workspaces per request +vws.group.authz.maxWorkspacesInGroup=1 -# not present means no restriction -#vws.group.authz.imageBaseDirectory= +# Image node +vws.group.authz.imageNodeHostname=example.com -# not present means false -#vws.group.authz.dirHashMode= +# Image node basedir +vws.group.authz.imageBaseDirectory=/cloud + +# Use DN hashing convention for user-specific subdirectories (see documentation) +vws.group.authz.dirHashMode=true From 93fb60d6c2df1b09cfc6f0f3b5bd65964d22add7 Mon Sep 17 00:00:00 2001 From: David LaBissoniere Date: Fri, 24 Jun 2011 11:58:27 -0500 Subject: [PATCH 21/21] Better default group-authz configuration Got rid of group setting for users in make-test-env.sh --- .../workspace-service/group-authz/README.txt | 2 + .../group-authz/group01.properties | 28 ++++++++------ .../group-authz/group02.properties | 6 --- .../group-authz/group03.properties | 6 --- .../group-authz/group04.properties | 37 +++++-------------- tests/make-test-env.sh | 4 +- 6 files changed, 29 insertions(+), 54 deletions(-) diff --git a/service/service/java/source/etc/workspace-service/group-authz/README.txt b/service/service/java/source/etc/workspace-service/group-authz/README.txt index 848cae47..e4c0e5ea 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/README.txt +++ b/service/service/java/source/etc/workspace-service/group-authz/README.txt @@ -9,6 +9,8 @@ in the "group01.txt" file (and so forth). You can make multiple kinds of allocations in this manner, dropping people into different authorization groups as you add them to the cloud. +You should use the nimbus-edit-user tool to change a user's group membership. + You can have 1-15 groups (and zero groups, but that is just like disabling the plugin entirely). diff --git a/service/service/java/source/etc/workspace-service/group-authz/group01.properties b/service/service/java/source/etc/workspace-service/group-authz/group01.properties index 18c04483..94c5bdbd 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group01.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group01.properties @@ -1,22 +1,26 @@ +# For logging and other side-purposes vws.group.authz.groupname=UNLIMITED # 0 means no restriction + +# Maximum currently reserved minutes at one point in time +# If the caller has two other workspaces with 600 minutes scheduled +# for each, the value being checked against this policy would be 1200 minutes +# plus whatever time the current request is for. vws.group.authz.maxReservedMinutes=0 -# 0 means no restriction +# Maximum elapsed (finished) and currently reserved minutes at one point in time +# If the caller has one other workspace with 600 minutes scheduled and 6000 +# minutes of recorded past usage, the value being checked against this policy +# would be 6600 minutes plus whatever time the current request is for. This is +# the all-time maximum usage cap. vws.group.authz.maxElapsedReservedMinutes=0 -# 0 means no restriction +# Maximum number of CPUs per VM instance +#vws.group.authz.maxCPUs=2 + +# Maximum number of running workspaces at one point in time vws.group.authz.maxWorkspaceNumber=0 -# 0 means no restriction +# Maximum number of workspaces per request vws.group.authz.maxWorkspacesInGroup=0 - -# not present means no restriction -#vws.group.authz.imageNodeHostname= - -# not present means no restriction -#vws.group.authz.imageBaseDirectory= - -# not present means false -#vws.group.authz.dirHashMode= diff --git a/service/service/java/source/etc/workspace-service/group-authz/group02.properties b/service/service/java/source/etc/workspace-service/group-authz/group02.properties index 1396a789..9682beb7 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group02.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group02.properties @@ -7,9 +7,3 @@ vws.group.authz.maxElapsedReservedMinutes=20160 vws.group.authz.maxWorkspaceNumber=5 vws.group.authz.maxWorkspacesInGroup=5 - -vws.group.authz.imageNodeHostname=example.com - -vws.group.authz.imageBaseDirectory=/cloud - -vws.group.authz.dirHashMode=true diff --git a/service/service/java/source/etc/workspace-service/group-authz/group03.properties b/service/service/java/source/etc/workspace-service/group-authz/group03.properties index e361cd82..08e93728 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group03.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group03.properties @@ -7,9 +7,3 @@ vws.group.authz.maxElapsedReservedMinutes=0 vws.group.authz.maxWorkspaceNumber=16 vws.group.authz.maxWorkspacesInGroup=16 - -vws.group.authz.imageNodeHostname=example.com - -vws.group.authz.imageBaseDirectory=/cloud - -vws.group.authz.dirHashMode=true diff --git a/service/service/java/source/etc/workspace-service/group-authz/group04.properties b/service/service/java/source/etc/workspace-service/group-authz/group04.properties index 9b5c7904..5d879e38 100644 --- a/service/service/java/source/etc/workspace-service/group-authz/group04.properties +++ b/service/service/java/source/etc/workspace-service/group-authz/group04.properties @@ -1,33 +1,14 @@ -# For logging and other side-purposes -vws.group.authz.groupname=TESTING +vws.group.authz.groupname=UNLIMITED -# Maximum currently reserved minutes at one point in time -# If the caller has two other workspaces with 600 minutes scheduled -# for each, the value being checked against this policy would be 1200 minutes -# plus whatever time the current request is for. -vws.group.authz.maxReservedMinutes=300 +# 0 means no restriction +vws.group.authz.maxReservedMinutes=0 -# Maximum elapsed (finished) and currently reserved minutes at one point in time -# If the caller has one other workspace with 600 minutes scheduled and 6000 -# minutes of recorded past usage, the value being checked against this policy -# would be 6600 minutes plus whatever time the current request is for. This is -# the all-time maximum usage cap. -vws.group.authz.maxElapsedReservedMinutes=300 +# 0 means no restriction +vws.group.authz.maxElapsedReservedMinutes=0 -# Maximum number of CPUs per VM instance -vws.group.authz.maxCPUs=2 +# 0 means no restriction +vws.group.authz.maxWorkspaceNumber=0 -# Maximum number of running workspaces at one point in time -vws.group.authz.maxWorkspaceNumber=5 +# 0 means no restriction +vws.group.authz.maxWorkspacesInGroup=0 -# Maximum number of workspaces per request -vws.group.authz.maxWorkspacesInGroup=1 - -# Image node -vws.group.authz.imageNodeHostname=example.com - -# Image node basedir -vws.group.authz.imageBaseDirectory=/cloud - -# Use DN hashing convention for user-specific subdirectories (see documentation) -vws.group.authz.dirHashMode=true diff --git a/tests/make-test-env.sh b/tests/make-test-env.sh index f8527e59..505a61ba 100755 --- a/tests/make-test-env.sh +++ b/tests/make-test-env.sh @@ -177,7 +177,7 @@ echo "=========================================" echo "Making a common user" echo "=========================================" user_name="nimbus@$RANDOM" -user_stuff=`$install_dir/bin/nimbus-new-user --group 04 --batch -r cloud_properties,cert,key,access_id,access_secret $user_name` +user_stuff=`$install_dir/bin/nimbus-new-user --batch -r cloud_properties,cert,key,access_id,access_secret $user_name` aid=`echo $user_stuff | awk -F , '{ print $4 }'` apw=`echo $user_stuff | awk -F , '{ print $5 }'` @@ -189,7 +189,7 @@ echo "Making a new user" echo "=========================================" user_name="nimbus@$RANDOM" -user_stuff=`$install_dir/bin/nimbus-new-user --group 04 --batch -r cloud_properties,cert,key,access_id,access_secret,canonical_id $user_name` +user_stuff=`$install_dir/bin/nimbus-new-user --batch -r cloud_properties,cert,key,access_id,access_secret,canonical_id $user_name` echo $user_stuff cp=`echo $user_stuff | awk -F , '{ print $1 }'`