diff --git a/bin/misspell b/bin/misspell new file mode 100755 index 000000000000..9fe371346aad Binary files /dev/null and b/bin/misspell differ diff --git a/install-misspell.sh b/install-misspell.sh new file mode 100644 index 000000000000..e24a84a20b36 --- /dev/null +++ b/install-misspell.sh @@ -0,0 +1,362 @@ +#!/bin/sh +set -e +# Code generated by godownloader. DO NOT EDIT. +# + +usage() { + this=$1 + cat </dev/null +} +echoerr() { + echo "$@" 1>&2 +} +log_prefix() { + echo "$0" +} +_logp=6 +log_set_priority() { + _logp="$1" +} +log_priority() { + if test -z "$1"; then + echo "$_logp" + return + fi + [ "$1" -ge "$_logp" ] +} +log_debug() { + log_priority 7 && echoerr "$(log_prefix)" "DEBUG" "$@" +} +log_info() { + log_priority 6 && echoerr "$(log_prefix)" "INFO" "$@" +} +log_err() { + log_priority 3 && echoerr "$(log_prefix)" "ERR" "$@" +} +log_crit() { + log_priority 2 && echoerr "$(log_prefix)" "CRIT" "$@" +} +uname_os() { + os=$(uname -s | tr '[:upper:]' '[:lower:]') + case "$os" in + msys_nt) os="windows" ;; + esac + echo "$os" +} +uname_arch() { + arch=$(uname -m) + case $arch in + x86_64) arch="amd64" ;; + x86) arch="386" ;; + i686) arch="386" ;; + i386) arch="386" ;; + aarch64) arch="arm64" ;; + armv5*) arch="arm5" ;; + armv6*) arch="arm6" ;; + armv7*) arch="arm7" ;; + esac + echo ${arch} +} +uname_os_check() { + os=$(uname_os) + case "$os" in + darwin) return 0 ;; + dragonfly) return 0 ;; + freebsd) return 0 ;; + linux) return 0 ;; + android) return 0 ;; + nacl) return 0 ;; + netbsd) return 0 ;; + openbsd) return 0 ;; + plan9) return 0 ;; + solaris) return 0 ;; + windows) return 0 ;; + esac + log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" + return 1 +} +uname_arch_check() { + arch=$(uname_arch) + case "$arch" in + 386) return 0 ;; + amd64) return 0 ;; + arm64) return 0 ;; + armv5) return 0 ;; + armv6) return 0 ;; + armv7) return 0 ;; + ppc64) return 0 ;; + ppc64le) return 0 ;; + mips) return 0 ;; + mipsle) return 0 ;; + mips64) return 0 ;; + mips64le) return 0 ;; + s390x) return 0 ;; + amd64p32) return 0 ;; + esac + log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" + return 1 +} +untar() { + tarball=$1 + case "${tarball}" in + *.tar.gz | *.tgz) tar -xzf "${tarball}" ;; + *.tar) tar -xf "${tarball}" ;; + *.zip) unzip "${tarball}" ;; + *) + log_err "untar unknown archive format for ${tarball}" + return 1 + ;; + esac +} +mktmpdir() { + test -z "$TMPDIR" && TMPDIR="$(mktemp -d)" + mkdir -p "${TMPDIR}" + echo "${TMPDIR}" +} +http_download() { + local_file=$1 + source_url=$2 + header=$3 + headerflag='' + destflag='' + if is_command curl; then + cmd='curl --fail -sSL' + destflag='-o' + headerflag='-H' + elif is_command wget; then + cmd='wget -q' + destflag='-O' + headerflag='--header' + else + log_crit "http_download unable to find wget or curl" + return 1 + fi + if [ -z "$header" ]; then + $cmd $destflag "$local_file" "$source_url" + else + $cmd $headerflag "$header" $destflag "$local_file" "$source_url" + fi +} +github_api() { + local_file=$1 + source_url=$2 + header="" + case "$source_url" in + https://api.github.com*) + test -z "$GITHUB_TOKEN" || header="Authorization: token $GITHUB_TOKEN" + ;; + esac + http_download "$local_file" "$source_url" "$header" +} +github_last_release() { + owner_repo=$1 + version=$2 + test -z "$version" && version="latest" + giturl="https://github.com/${owner_repo}/releases/${version}" + json=$(http_download "-" "$giturl" "Accept:application/json") + version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') + test -z "$version" && return 1 + echo "$version" +} +hash_sha256() { + TARGET=${1:-/dev/stdin} + if is_command gsha256sum; then + hash=$(gsha256sum "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command sha256sum; then + hash=$(sha256sum "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command shasum; then + hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 + echo "$hash" | cut -d ' ' -f 1 + elif is_command openssl; then + hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 + echo "$hash" | cut -d ' ' -f a + else + log_crit "hash_sha256 unable to find command to compute sha-256 hash" + return 1 + fi +} +hash_sha256_verify() { + TARGET=$1 + checksums=$2 + if [ -z "$checksums" ]; then + log_err "hash_sha256_verify checksum file not specified in arg2" + return 1 + fi + BASENAME=${TARGET##*/} + want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) + if [ -z "$want" ]; then + log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" + return 1 + fi + got=$(hash_sha256 "$TARGET") + if [ "$want" != "$got" ]; then + log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" + return 1 + fi +} +cat /dev/null <> /dev/null\ && sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \ && sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \ && tdnf clean all + +RUN tdnf erase -y toybox && tdnf install -y util-linux RUN tdnf erase -y toybox && tdnf install -y util-linux diff --git a/src/adminserver/client/client.go b/src/adminserver/client/client.go index 0f486e693ab4..f0b2faa25476 100644 --- a/src/adminserver/client/client.go +++ b/src/adminserver/client/client.go @@ -20,6 +20,7 @@ import ( "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/systeminfo/imagestorage" ) @@ -68,13 +69,14 @@ func (c *client) Ping() error { if !strings.Contains(addr, ":") { addr = addr + ":80" } - + log.Errorf("Skip to ping url: %v" + addr) return utils.TestTCPConn(addr, 60, 2) } // GetCfgs ... func (c *client) GetCfgs() (map[string]interface{}, error) { url := c.baseURL + "/api/configs" + log.Errorf("Create config from: url:%v", url) cfgs := map[string]interface{}{} if err := c.client.Get(url, &cfgs); err != nil { return nil, err diff --git a/src/adminserver/systemcfg/encrypt/encrypt.go b/src/adminserver/systemcfg/encrypt/encrypt.go index 8861e7862562..49c19882f07a 100644 --- a/src/adminserver/systemcfg/encrypt/encrypt.go +++ b/src/adminserver/systemcfg/encrypt/encrypt.go @@ -15,7 +15,7 @@ package encrypt import ( - comcfg "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/config/encrypt" "github.com/goharbor/harbor/src/common/utils" ) @@ -29,12 +29,12 @@ type Encryptor interface { // AESEncryptor uses AES to encrypt or decrypt string type AESEncryptor struct { - keyProvider comcfg.KeyProvider + keyProvider encrypt.KeyProvider keyParams map[string]interface{} } // NewAESEncryptor returns an instance of an AESEncryptor -func NewAESEncryptor(keyProvider comcfg.KeyProvider, +func NewAESEncryptor(keyProvider encrypt.KeyProvider, keyParams map[string]interface{}) Encryptor { return &AESEncryptor{ keyProvider: keyProvider, diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 88be64f714c2..6247cef22e6b 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -27,7 +27,7 @@ import ( "github.com/goharbor/harbor/src/adminserver/systemcfg/store/encrypt" "github.com/goharbor/harbor/src/adminserver/systemcfg/store/json" "github.com/goharbor/harbor/src/common" - comcfg "github.com/goharbor/harbor/src/common/config" + commonencrypt "github.com/goharbor/harbor/src/common/config/encrypt" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" @@ -358,7 +358,7 @@ func initCfgStore() (err error) { log.Infof("the path of key used by key provider: %s", kp) encryptor := enpt.NewAESEncryptor( - comcfg.NewFileKeyProvider(kp), nil) + commonencrypt.NewFileKeyProvider(kp), nil) CfgStore = encrypt.NewCfgStore(encryptor, attrs, CfgStore) return nil diff --git a/src/common/config/client/db/configdriver.go b/src/common/config/client/db/configdriver.go new file mode 100644 index 000000000000..adc09e85bf2f --- /dev/null +++ b/src/common/config/client/db/configdriver.go @@ -0,0 +1,81 @@ +package db + +import ( + "sync" + + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" +) + +// ConfigureDriver - Retrieve configurations from database +type ConfigureDriver struct { + config.ConfigureStore +} + +// NewDBConfigureStore ... +func NewDBConfigureStore() *ConfigureDriver { + return NewDBConfigureStoreFromArray(config.ConfigList) +} + +// NewDBConfigureStoreFromArray ... +func NewDBConfigureStoreFromArray(items []config.Item) *ConfigureDriver { + cd := &ConfigureDriver{*config.NewConfigureStore()} + config.MetaData.InitMetaDataFromArray(items) + cd.InitFromArray(items) + return cd +} + +var instance *ConfigureDriver +var once sync.Once + +// GetConfigureDriverInstance - get instance of DB ConfigureDriver +func GetConfigureDriverInstance() *ConfigureDriver { + once.Do(func() { + instance = &ConfigureDriver{*config.NewConfigureStore()} + config.MetaData.InitMetaDataFromArray(config.ConfigList) + instance.InitFromArray(config.ConfigList) + }) + return instance +} + +// Load ... +// FIXME: refactor +func (cd *ConfigureDriver) Load() error { + cfgs := map[string]string{} + configEntries, err := dao.GetConfigEntries() + if err != nil { + return err + } + for _, item := range configEntries { + // ignore item can be relead from env + itemMetadata, err := config.MetaData.GetConfigMetaData(item.Key) + if err != nil { + log.Errorf("failed to GetConfigMetaData, key:%v, error:%v", item.Key, err) + continue + } + if itemMetadata.Reloadable { + continue + } + cfgs[item.Key] = item.Value + } + cd.LoadFromMap(cfgs) + return nil +} + +// Save ... +func (cd *ConfigureDriver) Save() error { + var configEntries []models.ConfigEntry + configValues, err := cd.GetAllSettings() + if err != nil { + return err + } + for _, v := range configValues { + var entry = new(models.ConfigEntry) + entry.Key = v.GetKey() + entry.Value = v.GetString() + configEntries = append(configEntries, *entry) + } + return dao.SaveConfigEntries(configEntries) +} diff --git a/src/common/config/client/db/configdriver_test.go b/src/common/config/client/db/configdriver_test.go new file mode 100644 index 000000000000..ac4611ad8d36 --- /dev/null +++ b/src/common/config/client/db/configdriver_test.go @@ -0,0 +1,173 @@ +package db + +import ( + "fmt" + "os" + "testing" + + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/utils/log" +) + +var testingMetaDataArray = []config.Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic", DefaultValue: "3"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic", DefaultValue: "cn=admin,dc=example,dc=com"}, + {Name: "ldap_search_password", Type: "password", Scope: "user", Group: "ldapbasic"}, + {Name: "ulimit", Type: "int64", Scope: "user", Group: "ldapbasic", DefaultValue: "99999"}, + {Name: "ldap_verify_cert", Type: "bool", Scope: "user", Group: "ldapbasic", DefaultValue: "true"}, + {Name: "sample_map_setting", Type: "map", Scope: "user", Group: "ldapbasic"}, +} + +var testRegistryURL = "http://vmware.com:5000" + +func TestMain(m *testing.M) { + os.Setenv("REGISTRY_URL", testRegistryURL) + databases := []string{"postgresql"} + for _, database := range databases { + log.Infof("run test cases for database: %s", database) + result := 1 + switch database { + case "postgresql": + dao.PrepareTestForPostgresSQL() + default: + log.Fatalf("invalid database: %s", database) + } + result = testForAll(m) + + if result != 0 { + os.Exit(result) + } + } +} + +func testForAll(m *testing.M) int { + + rc := m.Run() + clearAll() + return rc +} + +func clearAll() { + tables := []string{"project_member", + "project_metadata", "access_log", "repository", "replication_policy", + "replication_target", "replication_job", "replication_immediate_trigger", "img_scan_job", + "img_scan_overview", "clair_vuln_timestamp", "project", "harbor_user"} + for _, t := range tables { + if err := dao.ClearTable(t); err != nil { + log.Errorf("Failed to clear table: %s,error: %v", t, err) + } + } +} + +func TestDBDriver_Load(t *testing.T) { + cd := NewDBConfigureStore() + cd.InitFromArray(testingMetaDataArray) + cd.Load() + cfgValue, err := cd.GetSettingByGroup("ldapbasic") + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + for _, item := range cfgValue { + fmt.Printf("config value is %+v", item.GetString()) + } +} +func TestDBDriver_Password(t *testing.T) { + os.Setenv("POSTGRESQL_USERNAME", "postgres") + os.Setenv("POSTGRESQL_PASSWORD", "root123") + os.Setenv("POSTGRESQL_SSLMODE", "disable") + os.Setenv("DATABASE_TYPE", "postgresql") + cd := NewDBConfigureStore() + cd.Init() + cd.Load() + fmt.Printf("got password %v", cd.GetString("postgresql_password")) + +} + +func TestDBDriver_Save(t *testing.T) { + cd := NewDBConfigureStore() + config.MetaData.InitMetaDataFromArray(testingMetaDataArray) + cd.InitFromArray(testingMetaDataArray) + cd.Load() + err := cd.UpdateConfigValue("ldap_search_dn", "cn=administrator,dc=vmware,dc=com") + if err != nil { + t.Errorf("Error occurred when UpdateConfigValue: %v", err) + } + err = cd.UpdateConfigValue("ldap_verify_cert", "F") + if err != nil { + t.Errorf("Error occurred when UpdateConfigValue : %v", err) + } + err = cd.UpdateConfigValue("ldap_search_scope", "2") + if err != nil { + t.Errorf("Error occurred when UpdateConfigValue: %v", err) + } + err = cd.UpdateConfigValue("ldap_search_password", "zhu88jie") + if err != nil { + t.Errorf("Error occurred when UpdateConfigureValue: %v", err) + } + + cd.Save() + cd.Load() + value, err := cd.GetSetting("ldap_search_password") + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + plainPasswd := value.GetPassword() + if plainPasswd != "zhu88jie" { + t.Error("Failed to get password") + } +} + +func TestCoreConfigManager_Load(t *testing.T) { + ccm := NewCoreConfigManagerFromArray(testingMetaDataArray) + cfgMap, err := ccm.Load() + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + if len(cfgMap) < 1 { + t.Error("Can not find any item in map") + } +} + +func TestCoreConfigManager_Upload(t *testing.T) { + ccm := NewCoreConfigManagerFromArray(testingMetaDataArray) + configMap := map[string]interface{}{} + configMap["ldap_search_dn"] = "cn=admin,dc=vmware,dc=com" + err := ccm.Upload(configMap) + if err != nil { + t.Errorf("Error occurred when Upload Config: %v", err) + } +} + +func TestConfigDriver(t *testing.T) { + configDriver := GetConfigureDriverInstance() + values, err := configDriver.GetAllSettings() + settingCount1 := len(values) + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + for _, item := range values { + fmt.Printf("key=%v, value=%v\n", item.GetKey(), item.GetString()) + } + configDriver.Load() + values, err = configDriver.GetAllSettings() + settingCount2 := len(values) + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + for _, item := range values { + fmt.Printf("key=%v, value=%v\n", item.GetKey(), item.GetString()) + } + + if settingCount1 >= settingCount2 { + t.Error("The count of setting after reload should be more") + } + +} +func TestConfigDriverWithEnv(t *testing.T) { + configDriver := GetConfigureDriverInstance() + registryURL := configDriver.GetString("registry_url") + if registryURL != testRegistryURL { + t.Errorf("Failed to set registry url in env") + } +} diff --git a/src/common/config/client/db/configmanager.go b/src/common/config/client/db/configmanager.go new file mode 100644 index 000000000000..fd379d40c075 --- /dev/null +++ b/src/common/config/client/db/configmanager.go @@ -0,0 +1,86 @@ +package db + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/utils/log" +) + +// CoreConfigManager ... Wrap the configure driver to previous interface, used for remove adminserver container only +type CoreConfigManager struct { + Driver *ConfigureDriver +} + +// NewCoreConfigManager ... +func NewCoreConfigManager() *CoreConfigManager { + return &CoreConfigManager{Driver: GetConfigureDriverInstance()} +} + +// NewCoreConfigManagerFromArray ... +func NewCoreConfigManagerFromArray(items []config.Item) *CoreConfigManager { + return &CoreConfigManager{Driver: NewDBConfigureStoreFromArray(items)} +} + +// Load ... +func (ccm *CoreConfigManager) Load() (map[string]interface{}, error) { + resultMap := map[string]interface{}{} + values, err := ccm.Driver.GetAllSettings() + if err != nil { + return resultMap, err + } + for _, item := range values { + key := item.GetKey() + itemMetaData, err := config.MetaData.GetConfigMetaData(key) + if err != nil { + log.Errorf("Can not get the metadata of current key:%v", key) + continue + } + if itemMetaData.Type == config.StringType { + resultMap[key] = item.GetString() + } else if itemMetaData.Type == config.IntType { + resultMap[key] = item.GetInt() + } else if itemMetaData.Type == config.Int64Type { + resultMap[key] = item.GetInt64() + } else if itemMetaData.Type == config.BoolType { + resultMap[key] = item.GetBool() + } else if itemMetaData.Type == config.PasswordType { + resultMap[key] = item.GetPassword() + } + } + return resultMap, nil +} + +// Get ... no cache temporary +func (ccm *CoreConfigManager) Get() (map[string]interface{}, error) { + return ccm.Load() +} + +// Upload ... +func (ccm *CoreConfigManager) Upload(cfgs map[string]interface{}) error { + for key, value := range cfgs { + err := ccm.Driver.UpdateConfigValue(key, fmt.Sprintf("%v", value)) + if err != nil { + log.Errorf("Failed to update configure key %v, value %v, error %v", key, value, err) + } + } + return ccm.Driver.Save() +} + +// Reset ... +func (ccm *CoreConfigManager) Reset() error { + ccm.Driver.Reset() + return nil +} + +// InitDatabaseAndConfigure - Initial database and configure +func InitDatabaseAndConfigure() { + configManager := NewCoreConfigManager() + database := configManager.Driver.GetDatabaseCfg() + dao.PrepareDatabase(database) + if err := configManager.Driver.Save(); err != nil { + log.Fatalf("failed to save configuration: %v", err) + } + GetConfigureDriverInstance().Load() +} diff --git a/src/common/config/client/db/configmanager_test.go b/src/common/config/client/db/configmanager_test.go new file mode 100644 index 000000000000..fc2011b72855 --- /dev/null +++ b/src/common/config/client/db/configmanager_test.go @@ -0,0 +1,39 @@ +package db + +import ( + "fmt" + "strings" + "testing" + + "github.com/goharbor/harbor/src/common" +) + +func TestConfigManagerLoad(t *testing.T) { + cfgManager := NewCoreConfigManager() + cfgManager.Upload(common.TestServerDefaultConfig) + cfg, err := cfgManager.Get() + if err != nil { + t.Errorf("Error occurred when : %v", err) + } + for key, value := range cfg { + if strings.HasPrefix(key, "ldap") { + fmt.Printf("message need to print,key=%v, value=%v\n", key, value) + } + } +} + +func TestConfigManagerUAA(t *testing.T) { + cfgManager := NewCoreConfigManager() + GetConfigureDriverInstance().Load() + cfg, err := cfgManager.Get() + fmt.Printf("message need to print,%+v\n", cfg) + if err != nil { + t.Errorf("Error occurred when Get config: %v", err) + } + if _, ok := cfg["uaa_endpoint"]; !ok { + t.Error("uaa_endpoint setting is wrong") + } + if _, ok := cfg["uaa_client_id"]; !ok { + t.Error("uaa_client_id setting is wrong") + } +} diff --git a/src/common/config/client/inmemory/configdriver.go b/src/common/config/client/inmemory/configdriver.go new file mode 100644 index 000000000000..8691c04c3550 --- /dev/null +++ b/src/common/config/client/inmemory/configdriver.go @@ -0,0 +1,21 @@ +package inmemory + +import "github.com/goharbor/harbor/src/common/config" + +// ConfigInMemory - used in testing only +type ConfigInMemory struct { + config.ConfigureStore +} + +// NewConfigInMemory ... +func NewConfigInMemory() *ConfigInMemory { + return NewConfigInMemoryFromArray(config.ConfigList) +} + +// NewConfigInMemoryFromArray ... +func NewConfigInMemoryFromArray(items []config.Item) *ConfigInMemory { + cim := &ConfigInMemory{*config.NewConfigureStore()} + config.MetaData.InitMetaDataFromArray(items) + cim.InitFromArray(items) + return cim +} diff --git a/src/common/config/client/inmemory/configdriver_test.go b/src/common/config/client/inmemory/configdriver_test.go new file mode 100644 index 000000000000..38f5f9b23315 --- /dev/null +++ b/src/common/config/client/inmemory/configdriver_test.go @@ -0,0 +1,104 @@ +package inmemory + +import ( + "fmt" + "os" + "testing" + + "github.com/goharbor/harbor/src/common/config" +) + +var testingMetaDataArray = []config.Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic", DefaultValue: "3"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic", DefaultValue: "cn=admin,dc=example,dc=com"}, + {Name: "ulimit", Type: "int64", Scope: "user", Group: "ldapbasic", DefaultValue: "99999"}, + {Name: "ldap_verify_cert", Type: "bool", Scope: "user", Group: "ldapbasic", DefaultValue: "true"}, + {Name: "sample_map_setting", Type: "map", Scope: "user", Group: "ldapbasic"}, +} + +func TestCreateInMemoryConfigInit(t *testing.T) { + + cfg := NewConfigInMemoryFromArray(testingMetaDataArray) + values, err := cfg.GetSettingByGroup("ldapbasic") + if err != nil { + t.Errorf("Error occurred when GetSettingByGroup: %v", err) + } + if len(values) != 4 { + t.Error("No keys in memory config") + } + for _, value := range values { + fmt.Printf("values %+v", value) + } + + userCfg, err := cfg.GetSettingByScope("user") + if err != nil || len(userCfg) != 3 { + t.Error("user setting config failed!") + } + +} + +func TestCreateInMemoryConfigSet(t *testing.T) { + var testingMetaDataArray = []config.Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic", DefaultValue: "3"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic", DefaultValue: "cn=admin,dc=example,dc=com"}, + {Name: "ulimit", Type: "int64", Scope: "user", Group: "ldapbasic", DefaultValue: "99999"}, + {Name: "ldap_verify_cert", Type: "bool", Scope: "user", Group: "ldapbasic", DefaultValue: "true"}, + {Name: "sample_map_setting", Type: "map", Scope: "user", Group: "ldapbasic"}, + } + cfg := NewConfigInMemoryFromArray(testingMetaDataArray) + err := cfg.UpdateConfigValue("ldap_search_dn", "cn=test,dc=example,dc=com") + if err != nil { + t.Errorf("Error occurred when UpdateConfigValue: %v", err) + } + value, err := cfg.GetSetting("ldap_search_dn") + if err != nil { + t.Errorf("Error occurred when GetSetting: %v", err) + } + str := value.GetString() + if err != nil { + t.Errorf("Error occurred when GetString: %v", err) + } + if str != "cn=test,dc=example,dc=com" { + t.Errorf("The get value is invalid") + } + fmt.Printf("the setting value is %v", str) + + ret := value.GetBool() + if ret != false { + t.Error("Should not convert string to bool!") + } + + ret2 := value.GetInt() + if ret2 != 0 { + t.Error("Should not convert string to integer!") + } + + ulimit, err := cfg.GetSetting("ulimit") + if err != nil { + t.Errorf("Error occurred when get ulimit: %v", err) + } + if ulimit.GetInt64() != 99999 { + t.Error("Failed to set ulimit") + } + +} + +func TestSetSystemSettings(t *testing.T) { + var testingMetaDataArray = []config.Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic", DefaultValue: "3"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic", DefaultValue: "cn=admin,dc=example,dc=com"}, + {Name: "ulimit", Type: "int64", Scope: "user", Group: "ldapbasic", DefaultValue: "99999"}, + {Name: "ldap_verify_cert", Type: "bool", Scope: "user", Group: "ldapbasic", DefaultValue: "true"}, + {Name: "sample_map_setting", Type: "map", Scope: "user", Group: "ldapbasic"}, + {Name: "auth_mode", Type: "string", Scope: "system", Group: "basic", EnvironmentKey: "AUTH_MODE", DefaultValue: "ldap_auth"}, + } + os.Setenv("AUTH_MODE", "db_auth") + cfg := NewConfigInMemoryFromArray(testingMetaDataArray) + value, err := cfg.GetSetting("auth_mode") + if err != nil { + t.Errorf("Error occurred when GetSetting: %v", err) + } + if value.GetString() != "db_auth" { + t.Errorf("The auth_mode is not set by environment !") + } +} diff --git a/src/common/config/client/remote/configdriver.go b/src/common/config/client/remote/configdriver.go new file mode 100644 index 000000000000..83ed3031566f --- /dev/null +++ b/src/common/config/client/remote/configdriver.go @@ -0,0 +1,80 @@ +package remote + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/http/modifier/auth" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" +) + +// ConfigureDriver - use http://core:8080/api/configurations to manage configuration, +// commonly used outside core api container +type ConfigureDriver struct { + config.ConfigureStore + // ConfigURL -- URL of configure server + ConfigURL string + httpClient *http.Client +} + +// Client - only used for remote client +type Client interface { + GetDatabaseCfg() (*models.Database, error) + GetCfgs() (map[string]interface{}, error) +} + +// Config contains configurations needed for client +type Config struct { + Secret string +} + +// NewRemoteConfigDriver ... Create a Remote Configure Driver +func NewRemoteConfigDriver(coreurl string, cfg *Config) (*ConfigureDriver, error) { + remoteDriver := &ConfigureDriver{ConfigURL: coreurl} + config.MetaData.InitMetaDataFromArray(config.ConfigList) + if cfg != nil { + authorizer := auth.NewSecretAuthorizer(cfg.Secret) + log.Errorf("The jobservice secret is %v, url:%v", cfg.Secret, remoteDriver.ConfigURL) + remoteDriver.httpClient = http.NewClient(nil, authorizer) + } else { + log.Error("Config is nil") + } + return remoteDriver, nil +} + +// Load ... load configures from URL +func (cd *ConfigureDriver) Load() error { + + url := cd.ConfigURL + "/api/configs" + + cfgs := map[string]interface{}{} + + if err := cd.httpClient.Get(url, &cfgs); err != nil { + return err + } + stringCfgs := map[string]string{} + for k, v := range cfgs { + stringCfgs[k] = fmt.Sprintf("%v", v) + } + // Get all configure entry from configure store + cd.LoadFromMap(stringCfgs) + return nil +} + +// GetDatabaseCfg ... Get database configure +func (cd *ConfigureDriver) GetDatabaseCfg() (*models.Database, error) { + if err := cd.Load(); err != nil { + return nil, err + } + return cd.ConfigureStore.GetDatabaseCfg(), nil +} + +// GetCfgs ... +func (cd *ConfigureDriver) GetCfgs() (map[string]interface{}, error) { + if err := cd.Load(); err != nil { + return nil, err + } + return cd.GetCfgs() +} diff --git a/src/common/config/config.go b/src/common/config/config.go index 90ad2c80fffb..cb97269bc050 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -32,6 +32,15 @@ type Manager struct { key string } +// ManagerInterface api for configure +type ManagerInterface interface { + Load() (map[string]interface{}, error) + Get() (map[string]interface{}, error) + Upload(cfgs map[string]interface{}) error + Reset() error + // Init() error +} + // NewManager returns an instance of Manager func NewManager(client client.Client, enableCache bool) *Manager { m := &Manager{ diff --git a/src/common/config/config_test.go b/src/common/config/config_test.go index 4fe7e29ecf72..5e3b7fbff4b4 100644 --- a/src/common/config/config_test.go +++ b/src/common/config/config_test.go @@ -13,5 +13,48 @@ // limitations under the License. package config +import "testing" + // the functions in common/config/config.go have been tested // by cases in UI and Jobservice + +func TestInitMetaData(t *testing.T) { + + MetaData.InitMetaData() + if item, err := MetaData.GetConfigMetaData("ldap_base_dn"); err != nil { + t.Error("Failed to find key ldap_search_base_dn after initial") + } else { + if item.Type != StringType { + t.Error("Wrong Type for this item!") + } + } + if item, err := MetaData.GetConfigMetaData("ldap_url"); err != nil { + t.Error("Failed to find key ldap_search_base_dn after initial") + } else { + if item.Type != StringType { + t.Error("Wrong Type for this item!") + } + } + if item, err := MetaData.GetConfigMetaData("ldap_scope"); err != nil { + t.Error("Failed to find key ldap_search_scope after initial") + } else { + if item.Type != IntType { + t.Error("Wrong Type for this item!") + } + } + if item, err := MetaData.GetConfigMetaData("ldap_search_password"); err != nil { + t.Error("Failed to find key ldap_search_password after initial") + } else { + if item.Type != PasswordType { + t.Error("Wrong Type for this item!") + } + } + if item, err := MetaData.GetConfigMetaData("ldap_verify_cert"); err != nil { + t.Error("Failed to find key ldap_verify_cert after initial") + } else { + if item.Type != BoolType { + t.Error("Wrong Type for this item!") + } + } + +} diff --git a/src/common/config/configclient.go b/src/common/config/configclient.go new file mode 100644 index 000000000000..9f17a7dda3c0 --- /dev/null +++ b/src/common/config/configclient.go @@ -0,0 +1,13 @@ +package config + +import "github.com/goharbor/harbor/src/common/models" + +// Client used to retrieve configuration +type Client interface { + GetSettingByGroup(groupName string) ([]Value, error) + GetSettingByScope(scope string) ([]Value, error) + GetSetting(keyName string) (Value, error) + UpdateConfig(cfg map[string]string) error + UpdateConfigValue(key string, value string) error + GetDatabaseCfg() *models.Database +} diff --git a/src/common/config/configlist.go b/src/common/config/configlist.go new file mode 100644 index 000000000000..34ac220f1325 --- /dev/null +++ b/src/common/config/configlist.go @@ -0,0 +1,106 @@ +package config + +import "github.com/goharbor/harbor/src/common" + +// Item - Configure item include default value, type, env name +type Item struct { + // The Scope of this configuration item: eg: SystemScope, UserScope + Scope string `json:"scope,omitempty"` + // email, ldapbasic, ldapgroup, uaa settings, used to retieve configure items by group, for example GetLDAPBasicSetting, GetLDAPGroupSetting settings + Group string `json:"group,omitempty"` + // environment key to retrieves this value when initialize, for example: POSTGRESQL_HOST, only used for system settings, for user settings no EnvironmentKey + EnvironmentKey string `json:"environment_key,omitempty"` + // The default string value for this key + DefaultValue string `json:"default_value,omitempty"` + // The key for current configure settings in database and rerest api + Name string `json:"name,omitempty"` + // It can be integer, string, bool, password, map + Type string `json:"type,omitempty"` + // The validation function for this field. + Validator ValidateFunc `json:"validator,omitempty"` + // Is this settign can be modified after configure + Editable bool `json:"editable,omitempty"` + // Reloadable - reload config from env after restart, if it is true, the setting is only reload from env + Reloadable bool `json:"reloadable,omitempty"` +} + +var ( + // ConfigList - All configure items used in harbor + // Steps to onboard a new setting + // 1. Add configure item in configlist.go + // 2. Get settings by config.Client + ConfigList = []Item{ + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "HARBOR_ADMIN_PASSWORD", DefaultValue: "", Name: "admin_initial_password", Type: PasswordType, Editable: true, Reloadable: false}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "ADMIRAL_URL", DefaultValue: "", Name: "admiral_url", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "AUTH_MODE", DefaultValue: "db_auth", Name: "auth_mode", Type: StringType, Editable: false}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "CFG_EXPIRATION", DefaultValue: "", Name: "cfg_expiration", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "CHART_REPOSITORY_URL", DefaultValue: "", Name: "chart_repository_url", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB", DefaultValue: "", Name: "clair_db", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB_HOST", DefaultValue: "", Name: "clair_db_host", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB_PASSWORD", DefaultValue: "", Name: "clair_db_password", Type: PasswordType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB_PORT", DefaultValue: "", Name: "clair_db_port", Type: IntType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB_SSLMODE", DefaultValue: "", Name: "clair_db_sslmode", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_DB_USERNAME", DefaultValue: "", Name: "clair_db_username", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: ClairGroup, EnvironmentKey: "CLAIR_URL", DefaultValue: "", Name: "clair_url", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "CORE_URL", DefaultValue: "", Name: "core_url", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "DATABASE_TYPE", DefaultValue: "postgresql", Name: "database_type", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_FROM", DefaultValue: "admin ", Name: "email_from", Type: StringType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_HOST", DefaultValue: "smtp.mydomain.com", Name: "email_host", Type: StringType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_IDENTITY", DefaultValue: "", Name: "email_identity", Type: StringType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_INSECURE", DefaultValue: "false", Name: "email_insecure", Type: BoolType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_PWD", DefaultValue: "", Name: "email_password", Type: PasswordType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_PORT", DefaultValue: "25", Name: "email_port", Type: IntType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_SSL", DefaultValue: "false", Name: "email_ssl", Type: BoolType, Editable: false}, + {Scope: UserScope, Group: EmailGroup, EnvironmentKey: "EMAIL_USR", DefaultValue: "sample_admin@mydomain.com", Name: "email_username", Type: StringType, Editable: false}, + + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "EXT_ENDPOINT", DefaultValue: "https://host01.com", Name: "ext_endpoint", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "JOBSERVICE_URL", DefaultValue: "", Name: "jobservice_url", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_BASE_DN", DefaultValue: "", Name: "ldap_base_dn", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_FILTER", DefaultValue: "", Name: "ldap_filter", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapGroupGroup, EnvironmentKey: "LDAP_GROUP_BASE_DN", DefaultValue: "", Name: "ldap_group_base_dn", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapGroupGroup, EnvironmentKey: "LDAP_GROUP_ADMIN_DN", DefaultValue: "", Name: "ldap_group_admin_dn", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapGroupGroup, EnvironmentKey: "LDAP_GROUP_GID", DefaultValue: "", Name: "ldap_group_attribute_name", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapGroupGroup, EnvironmentKey: "LDAP_GROUP_FILTER", DefaultValue: "", Name: "ldap_group_search_filter", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapGroupGroup, EnvironmentKey: "LDAP_GROUP_SCOPE", DefaultValue: "2", Name: "ldap_group_search_scope", Type: IntType, Editable: false}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_SCOPE", DefaultValue: "2", Name: "ldap_scope", Type: IntType, Editable: true}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_SEARCH_DN", DefaultValue: "", Name: "ldap_search_dn", Type: StringType, Editable: false}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_SEARCH_PWD", DefaultValue: "", Name: "ldap_search_password", Type: PasswordType, Editable: false}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_TIMEOUT", DefaultValue: "5", Name: "ldap_timeout", Type: IntType, Editable: false}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_UID", DefaultValue: "", Name: "ldap_uid", Type: StringType, Editable: true}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_URL", DefaultValue: "", Name: "ldap_url", Type: StringType, Editable: true}, + {Scope: UserScope, Group: LdapBasicGroup, EnvironmentKey: "LDAP_VERIFY_CERT", DefaultValue: "true", Name: "ldap_verify_cert", Type: BoolType, Editable: false}, + + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "MAX_JOB_WORKERS", DefaultValue: "", Name: "max_job_workers", Type: IntType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "NOTARY_URL", DefaultValue: "", Name: "notary_url", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_DATABASE", DefaultValue: "registry", Name: "postgresql_database", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_HOST", DefaultValue: "", Name: "postgresql_host", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_PASSWORD", DefaultValue: "root123", Name: "postgresql_password", Type: PasswordType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_PORT", DefaultValue: "5432", Name: "postgresql_port", Type: IntType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", Name: "postgresql_sslmode", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: DatabaseGroup, EnvironmentKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", Name: "postgresql_username", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, Name: "project_creation_restriction", Type: StringType, Editable: false}, + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "READ_ONLY", DefaultValue: "false", Name: "read_only", Type: BoolType, Editable: false}, + + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "REGISTRY_STORAGE_PROVIDER_NAME", DefaultValue: "filesystem", Name: "registry_storage_provider_name", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "REGISTRY_URL", DefaultValue: "http://registry:5000", Name: "registry_url", Type: StringType, Editable: false, Reloadable: true}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "REGISTRY_CONTROLLER_URL", DefaultValue: "http://registryctl:8080", Name: "registry_controller_url", Type: StringType, Editable: false, Reloadable: true}, + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "SELF_REGISTRATION", DefaultValue: "true", Name: "self_registration", Type: BoolType, Editable: false}, + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "TOKEN_EXPIRATION", DefaultValue: "30", Name: "token_expiration", Type: IntType, Editable: false}, + {Scope: SystemScope, Group: BasicGroup, EnvironmentKey: "TOKEN_SERVICE_URL", DefaultValue: "", Name: "token_service_url", Type: StringType, Editable: false, Reloadable: true}, + + {Scope: UserScope, Group: UAAGroup, EnvironmentKey: "UAA_CLIENTID", DefaultValue: "", Name: "uaa_client_id", Type: StringType, Editable: false}, + {Scope: UserScope, Group: UAAGroup, EnvironmentKey: "UAA_CLIENTSECRET", DefaultValue: "", Name: "uaa_client_secret", Type: StringType, Editable: false}, + {Scope: UserScope, Group: UAAGroup, EnvironmentKey: "UAA_ENDPOINT", DefaultValue: "", Name: "uaa_endpoint", Type: StringType, Editable: false}, + {Scope: UserScope, Group: UAAGroup, EnvironmentKey: "UAA_VERIFY_CERT", DefaultValue: "false", Name: "uaa_verify_cert", Type: BoolType, Editable: false}, + + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "WITH_CHARTMUSEUM", DefaultValue: "", Name: "with_chartmuseum", Type: BoolType, Editable: true, Reloadable: true}, + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "WITH_CLAIR", DefaultValue: "", Name: "with_clair", Type: BoolType, Editable: true, Reloadable: true}, + {Scope: UserScope, Group: BasicGroup, EnvironmentKey: "WITH_NOTARY", DefaultValue: "", Name: "with_notary", Type: BoolType, Editable: true, Reloadable: true}, + } +) diff --git a/src/common/config/configmetadata.go b/src/common/config/configmetadata.go new file mode 100644 index 000000000000..eb72d9613f13 --- /dev/null +++ b/src/common/config/configmetadata.go @@ -0,0 +1,92 @@ +package config + +import ( + "sync" +) + +// Constant for configure item +const ( + // Scope + UserScope = "user" + SystemScope = "system" + // Group + LdapBasicGroup = "ldapbasic" + LdapGroupGroup = "ldapgroup" + EmailGroup = "email" + UAAGroup = "uaa" + DatabaseGroup = "database" + BasicGroup = "basic" + ClairGroup = "clair" + + // Type + IntType = "int" + Int64Type = "int64" + StringType = "string" + BoolType = "bool" + PasswordType = "password" + MapType = "map" +) + +// ValidateFunc - function to validate configure items +type ValidateFunc func(key, value string) error + +// ConfigureMetaData ... +type ConfigureMetaData struct { + sync.RWMutex + metaMap map[string]Item +} + +// MetaData ... +var MetaData = NewConfigureMetaData() + +// NewConfigureMetaData ... +func NewConfigureMetaData() *ConfigureMetaData { + cm := new(ConfigureMetaData) + cm.metaMap = make(map[string]Item) + return cm +} + +func (cm *ConfigureMetaData) readMap(key string) (Item, error) { + cm.RLock() + defer cm.RUnlock() + if item, ok := cm.metaMap[key]; ok { + return item, nil + } + return Item{}, ErrNotDefined + +} +func (cm *ConfigureMetaData) writeMap(key string, item Item) { + cm.Lock() + defer cm.Unlock() + cm.metaMap[key] = item +} + +// InitMetaData ... +func (cm *ConfigureMetaData) InitMetaData() { + for _, item := range ConfigList { + cm.writeMap(item.Name, item) + } +} + +// InitMetaDataFromArray - used for testing +func (cm *ConfigureMetaData) InitMetaDataFromArray(items []Item) { + for _, item := range items { + cm.writeMap(item.Name, item) + } +} + +// GetAllConfigureItems - Get All Configuration Items +func (cm *ConfigureMetaData) GetAllConfigureItems() (items []Item) { + cm.RLock() + defer cm.RUnlock() + result := make([]Item, 0) + for _, item := range cm.metaMap { + result = append(result, item) + } + return result +} + +// GetConfigMetaData - Get single configuration item +func (cm *ConfigureMetaData) GetConfigMetaData(key string) (Item, error) { + return cm.readMap(key) +} diff --git a/src/common/config/configmetadata_test.go b/src/common/config/configmetadata_test.go new file mode 100644 index 000000000000..bbb48de69e31 --- /dev/null +++ b/src/common/config/configmetadata_test.go @@ -0,0 +1,29 @@ +package config + +import ( + "testing" +) + +func TestInitMetaDataFromJsonString(t *testing.T) { + + testConfigMetaData := []Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic"}, + } + MetaData.InitMetaDataFromArray(testConfigMetaData) + if item, err := MetaData.GetConfigMetaData("ldap_search_scope"); err != nil { + t.Error("failed to find ldap_search_scope!") + } else { + if item.Type != IntType { + t.Errorf("Failed to get the type,expect int, actual type %v", item.Type) + } + } + if item, err := MetaData.GetConfigMetaData("ldap_search_dn"); err != nil { + t.Error("failed to find ldap_search_dn!") + } else { + if item.Type != StringType { + t.Errorf("Failed to get the type string,expect string, actual type %v", item.Type) + } + } + +} diff --git a/src/common/config/configstore.go b/src/common/config/configstore.go new file mode 100644 index 000000000000..4810ab4411dd --- /dev/null +++ b/src/common/config/configstore.go @@ -0,0 +1,369 @@ +package config + +import ( + "os" + "sync" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" +) + +// ConfigureStore - to manage all configurations +type ConfigureStore struct { + sync.RWMutex + // ConfigureValues to store all configure values + configureValues sync.Map +} + +// NewConfigureStore ... +func NewConfigureStore() *ConfigureStore { + cs := new(ConfigureStore) + cs.configureValues = sync.Map{} + return cs +} + +func (s *ConfigureStore) readMap(key string) (Value, error) { + if value, ok := s.configureValues.Load(key); ok { + if result, suc := value.(Value); suc { + return result, nil + } + return nil, ErrTypeNotMatch + } + return nil, ErrValueNotSet + +} + +func (s *ConfigureStore) writeMap(key string, value Value) { + s.configureValues.Store(key, value) +} + +// StorageInterface ... +type StorageInterface interface { + // Init - init configurations with default value + Init() error + // InitFromString - used for testing + InitFromString(testingMetaDataArray []Item) error + // Load from store + Load() error + // Save to store + Save() error + // LoadFromMap ... + LoadFromMap(map[string]string) + // Save all configuration to store + UpdateAll() error + // Reset configure to default value + Reset() +} + +// Init - int the store +func (s *ConfigureStore) Init() error { + MetaData.InitMetaData() + // Init Default Value + itemArray := MetaData.GetAllConfigureItems() + for _, item := range itemArray { + if len(item.DefaultValue) > 0 { + c := &ConfigureValue{item.Name, item.DefaultValue} + err := c.Validate() + if err == nil { + s.writeMap(item.Name, c) + } else { + log.Errorf("Failed to init config item %+v, default err: %+v", c, err) + } + } + } + + // Init System Value + for _, item := range itemArray { + if item.Scope == SystemScope { + if len(item.EnvironmentKey) > 0 { + if envValue, ok := os.LookupEnv(item.EnvironmentKey); ok { + c := &ConfigureValue{item.Name, envValue} + err := c.Validate() + if err == nil { + s.writeMap(item.Name, c) + } else { + log.Errorf("Failed to init system config item %+v, err: %+v", c, err) + } + } + } + } + } + + return nil +} + +// LoadFromMap ... +func (s *ConfigureStore) LoadFromMap(cfgs map[string]string) { + for k, v := range cfgs { + c := &ConfigureValue{k, v} + err := c.Validate() + if err != nil { + log.Errorf("Failed LoadFromMap, config item %+v, err: %+v", c, err) + continue + } + s.writeMap(k, c) + } +} + +// InitFromArray ... Used for testing +func (s *ConfigureStore) InitFromArray(testingMetaDataArray []Item) error { + MetaData.InitMetaDataFromArray(testingMetaDataArray) + itemArray := MetaData.GetAllConfigureItems() + // Init Default Value + for _, item := range itemArray { + if len(item.DefaultValue) > 0 { + c := &ConfigureValue{item.Name, item.DefaultValue} + err := c.Validate() + if err == nil { + s.writeMap(item.Name, c) + } else { + log.Errorf("Failed InitFromArray, config item %+v, err: %+v", c, err) + } + } + } + + // Init System Value + for _, item := range itemArray { + if item.Scope == SystemScope { + if len(item.EnvironmentKey) > 0 { + if envValue, ok := os.LookupEnv(item.EnvironmentKey); ok { + c := &ConfigureValue{item.Name, envValue} + err := c.Validate() + if err == nil { + s.writeMap(item.Name, c) + } else { + log.Errorf("Failed InitFromArray, config item %+v, err: %+v", c, err) + } + } + } + } + } + + return nil +} + +// Load ... +func (s *ConfigureStore) Load() error { + panic("Load not implemented") +} + +// Save ... +func (s *ConfigureStore) Save() error { + panic("Save not implemented") +} + +// UpdateAll ... +func (s *ConfigureStore) UpdateAll() error { + log.Info("UpdateAll not implemented") + return nil +} + +// Reset ... +func (s *ConfigureStore) Reset() { + s.Lock() + defer s.Unlock() + err := s.Init() + if err != nil { + log.Errorf("Error occurred when Init: %v", err) + return + } + err = s.UpdateAll() + if err != nil { + log.Errorf("Error occurred when UpdateAll: %v", err) + } + +} + +// GetAllSettings ... +func (s *ConfigureStore) GetAllSettings() ([]Value, error) { + resultValues := make([]Value, 0) + s.configureValues.Range(func(key, value interface{}) bool { + if result, suc := value.(Value); suc { + resultValues = append(resultValues, result) + } + return true + }) + return resultValues, nil +} + +// GetSettingByGroup ... +func (s *ConfigureStore) GetSettingByGroup(groupName string) ([]Value, error) { + resultValues := make([]Value, 0) + s.configureValues.Range(func(key, value interface{}) bool { + if keyString, suc := key.(string); suc { + itemMataData, err := MetaData.GetConfigMetaData(keyString) + if err == nil { + if actValue, ok := value.(Value); ok && itemMataData.Group == groupName { + resultValues = append(resultValues, actValue) + } + } + } + return true + }) + + return resultValues, nil +} + +// GetSettingByScope ... +func (s *ConfigureStore) GetSettingByScope(scope string) ([]Value, error) { + resultValues := make([]Value, 0) + s.configureValues.Range(func(key, value interface{}) bool { + if keyString, suc := key.(string); suc { + itemMataData, err := MetaData.GetConfigMetaData(keyString) + if err == nil { + if actValue, ok := value.(Value); ok && itemMataData.Scope == scope { + resultValues = append(resultValues, actValue) + } + } + } + return true + }) + return resultValues, nil +} + +// GetSetting ... +func (s *ConfigureStore) GetSetting(keyName string) (Value, error) { + _, err := MetaData.GetConfigMetaData(keyName) + if err == nil { + return s.readMap(keyName) + } + return nil, ErrNotDefined +} + +// GetInt ... +func (s *ConfigureStore) GetInt(keyName string) int { + itemMetadata, err := MetaData.GetConfigMetaData(keyName) + if err == nil { + value, err := s.readMap(keyName) + if err != nil { + log.Errorf("Error while getting %v, error: %+v", keyName, err) + return 0 + } + if itemMetadata.Type == IntType { + return value.GetInt() + } + return 0 + } + return 0 +} + +// GetString ... +func (s *ConfigureStore) GetString(keyName string) string { + _, err := MetaData.GetConfigMetaData(keyName) + if err == nil { + value, err := s.readMap(keyName) + if err != nil { + log.Errorf("Error while getting %v, error: %+v", keyName, err) + return "" + } + return value.GetString() + } + return "" +} + +// GetPassword ... +func (s *ConfigureStore) GetPassword(keyName string) string { + itemMetadata, err := MetaData.GetConfigMetaData(keyName) + if err == nil { + value, err := s.readMap(keyName) + if err != nil { + log.Errorf("Error while geting %v, error: %+v", keyName, err) + return "" + } + if itemMetadata.Type == PasswordType { + return value.GetPassword() + } + } + return "" +} + +// GetBool ... +func (s *ConfigureStore) GetBool(keyName string) bool { + itemMetadata, err := MetaData.GetConfigMetaData(keyName) + if err == nil { + value, err := s.readMap(keyName) + if err != nil { + log.Errorf("Error while geting %v, error: %+v", keyName, err) + return false + } + if itemMetadata.Type == BoolType { + return value.GetBool() + } + } + return false +} + +// UpdateConfig ... +func (s *ConfigureStore) UpdateConfig(cfg map[string]string) error { + for key, value := range cfg { + err := s.UpdateConfigValue(key, value) + if err != nil { + return err + } + } + return nil +} + +// UpdateConfigValue ... +func (s *ConfigureStore) UpdateConfigValue(keyName string, value string) error { + itemMetadata, err := MetaData.GetConfigMetaData(keyName) + c := &ConfigureValue{Key: keyName, Value: value} + if err == nil { + if itemMetadata.Type == PasswordType { + err := c.Set(keyName, value) + if err != nil { + return err + } + } else { + err := c.Validate() + if err != nil { + return err + } + } + s.writeMap(keyName, c) + } + return err +} + +// GetDatabaseCfg ... +func (s *ConfigureStore) GetDatabaseCfg() *models.Database { + return &models.Database{ + Type: s.GetString(common.DatabaseType), + PostGreSQL: &models.PostGreSQL{ + Host: s.GetString(common.PostGreSQLHOST), + Port: s.GetInt(common.PostGreSQLPort), + Username: s.GetString(common.PostGreSQLUsername), + Password: s.GetPassword(common.PostGreSQLPassword), + Database: s.GetString(common.PostGreSQLDatabase), + SSLMode: s.GetString(common.PostGreSQLSSLMode), + }, + } +} + +// GetCfgs ... +func (s *ConfigureStore) GetCfgs() map[string]interface{} { + resultMap := map[string]interface{}{} + s.configureValues.Range(func(key, value interface{}) bool { + if result, suc := value.(Value); suc { + keyName := result.GetKey() + itemMetadata, err := MetaData.GetConfigMetaData(keyName) + if err != nil { + log.Errorf("Failed to get metadata for key %v", keyName) + } else if itemMetadata.Type == BoolType { + resultMap[keyName] = result.GetBool() + } else if itemMetadata.Type == IntType { + resultMap[keyName] = result.GetInt() + } else if itemMetadata.Type == Int64Type { + resultMap[keyName] = result.GetInt64() + } else if itemMetadata.Type == PasswordType { + resultMap[keyName] = result.GetString() + } else if itemMetadata.Type == StringType { + resultMap[keyName] = result.GetString() + } + } + return true + }) + return resultMap + +} diff --git a/src/common/config/configstore_test.go b/src/common/config/configstore_test.go new file mode 100644 index 000000000000..f85173da6b9c --- /dev/null +++ b/src/common/config/configstore_test.go @@ -0,0 +1,10 @@ +package config + +import ( + "fmt" + "testing" +) + +func TestInitConfigureStore(t *testing.T) { + fmt.Println("Testing message") +} diff --git a/src/common/config/configvalue.go b/src/common/config/configvalue.go new file mode 100644 index 000000000000..c667ad171645 --- /dev/null +++ b/src/common/config/configvalue.go @@ -0,0 +1,224 @@ +package config + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + + "github.com/goharbor/harbor/src/common/config/encrypt" + "github.com/goharbor/harbor/src/common/utils/log" +) + +var ( + // ErrNotDefined ... + ErrNotDefined = errors.New("configure item is not defined in metadata") + // ErrTypeNotMatch ... + ErrTypeNotMatch = errors.New("the required value doesn't matched with metadata defined") + // ErrInvalidData ... + ErrInvalidData = errors.New("the data provided is invalid") + // ErrValueNotSet ... + ErrValueNotSet = errors.New("the configure value is not set") +) + +// ConfigureValue - Configure values +type ConfigureValue struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +// Value -- interface to operate configure value +type Value interface { + GetString() string + // GetInt - return the int value of current value + GetInt() int + // GetInt64 - return the int64 value of current value + GetInt64() int64 + // GetBool - return the bool value of current setting + GetBool() bool + // GetStringToStringMap - return the string to string map of current value + GetStringToStringMap() map[string]string + // GetMap - return the map of current value + GetMap() map[string]interface{} + // Validator to validate configure items, if passed, return true, else return false and return error + Validate() error + // Set this configure item to configure store + Set(key, value string) error + + GetKey() string + + GetPassword() string + + SetPassword(key, password string) error +} + +// NewConfigureValue ... +func NewConfigureValue(key, value string) *ConfigureValue { + result := &ConfigureValue{} + if metaData, err := MetaData.GetConfigMetaData(key); err != nil { + if metaData.Type == PasswordType { + result.Set(key, value) + } else { + result.Set(key, value) + } + } else { + log.Errorf("Failed to create a configure value without metadata, key %v, value %v", key, value) + } + return result +} + +// GetString - Get the string value of current configure +func (c *ConfigureValue) GetString() string { + // Any type has the string value + _, err := MetaData.GetConfigMetaData(c.Key) + if err == nil { + return c.Value + } + return "" +} + +// GetKey ... +func (c *ConfigureValue) GetKey() string { + return c.Key +} + +// GetInt - return the int value of current value +func (c *ConfigureValue) GetInt() int { + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + if item.Type == IntType { + result, err := strconv.Atoi(c.Value) + if err == nil { + return result + } + } + } + log.Errorf("The current value can not convert to integer, %+v", c) + return 0 +} + +// GetInt64 - return the int64 value of current value +func (c *ConfigureValue) GetInt64() int64 { + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + if (item.Type == IntType) || (item.Type == Int64Type) { + result, err := strconv.ParseInt(c.Value, 10, 64) + if err == nil { + return result + } + } + } + log.Errorf("The current value can not convert to integer, %+v", c) + return 0 +} + +// GetBool - return the bool value of current setting +func (c *ConfigureValue) GetBool() bool { + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + if item.Type == BoolType { + result, err := strconv.ParseBool(c.Value) + if err == nil { + return result + } + } + } + log.Errorf("The current value can not convert to bool, %+v", c) + return false +} + +// GetStringToStringMap - return the string to string map of current value +func (c *ConfigureValue) GetStringToStringMap() map[string]string { + result := map[string]string{} + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + if item.Type == MapType { + err := json.Unmarshal([]byte(c.Value), &result) + if err == nil { + return result + } + } + } + log.Errorf("The current value can not convert to map[string]string, %+v", c) + return result +} + +// GetMap - return the map of current value +func (c *ConfigureValue) GetMap() map[string]interface{} { + result := map[string]interface{}{} + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + if item.Type == MapType { + err := json.Unmarshal([]byte(c.Value), &result) + if err == nil { + return result + } + } + } + log.Errorf("The current value can not convert to map[string]interface{}, %+v", c) + return result +} + +// Validate - to validate configure items, if passed, return true, else return false and return error +func (c *ConfigureValue) Validate() error { + if item, err := MetaData.GetConfigMetaData(c.Key); err == nil { + // Validate based on data type first + if item.Type == BoolType { + if strings.EqualFold(c.Value, "on") { + c.Value = "true" + } else if strings.EqualFold(c.Value, "off") { + c.Value = "false" + } + boolValue, err := strconv.ParseBool(c.Value) + if err != nil { + return ErrTypeNotMatch + } + c.Value = strconv.FormatBool(boolValue) + } else if item.Type == IntType { + if _, err := strconv.Atoi(c.Value); err != nil { + return ErrTypeNotMatch + } + } else if item.Type == Int64Type { + if _, err := strconv.ParseInt(c.Value, 10, 64); err != nil { + return ErrTypeNotMatch + } + } + + if item.Validator != nil { + return item.Validator(c.Key, c.Value) + } + return nil + } + return ErrNotDefined +} + +// GetPassword ... +func (c *ConfigureValue) GetPassword() string { + // value, err := encrypt.GetInstance().Decrypt(c.GetString()) + // if err != nil { + // log.Errorf("Failed to get password! key: %v, value %v, error %v", c.GetKey(), c.GetString(), err) + // return "" + // } + return c.GetString() +} + +// SetPassword ... +func (c *ConfigureValue) SetPassword(key, value string) error { + encryptedValue, err := encrypt.GetInstance().Encrypt(value) + log.Infof("Encrypt password %v, encrypted: %v ", value, encryptedValue) + if err != nil { + return err + } + return c.Set(key, encryptedValue) +} + +// Set - set this configure item to configure store +func (c *ConfigureValue) Set(key, value string) error { + if item, err := MetaData.GetConfigMetaData(key); err == nil { + if item.Validator != nil { + err := item.Validator(key, value) + if err != nil { + return ErrInvalidData + } + } + c.Key = key + c.Value = value + return nil + } + return ErrNotDefined +} diff --git a/src/common/config/configvalue_test.go b/src/common/config/configvalue_test.go new file mode 100644 index 000000000000..830e679fbb4e --- /dev/null +++ b/src/common/config/configvalue_test.go @@ -0,0 +1,240 @@ +package config + +import ( + "errors" + "reflect" + "testing" +) + +var testingMetaDataArray = []Item{ + {Name: "ldap_search_scope", Type: "int", Scope: "system", Group: "ldapbasic"}, + {Name: "ldap_search_dn", Type: "string", Scope: "user", Group: "ldapbasic"}, + {Name: "ulimit", Type: "int64", Scope: "user", Group: "ldapbasic"}, + {Name: "ldap_verify_cert", Type: "bool", Scope: "user", Group: "ldapbasic"}, + {Name: "sample_map_setting", Type: "map", Scope: "user", Group: "ldapbasic"}, +} + +func TestConfigureValue_GetString(t *testing.T) { + + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + want string + wantErr bool + }{ + {"normal", fields{"ldap_search_dn", "cn=admin,dc=example,dc=com"}, "cn=admin,dc=example,dc=com", false}, + } + + MetaData.InitMetaDataFromArray(testingMetaDataArray) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + got := c.GetString() + if got != tt.want { + t.Errorf("ConfigureValue.GetString() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfigureValue_GetInt64(t *testing.T) { + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + want int64 + wantErr bool + }{ + {"normal", fields{"ulimit", "255534223"}, 255534223, false}, + } + MetaData.InitMetaDataFromArray(testingMetaDataArray) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + got := c.GetInt64() + if got != tt.want { + t.Errorf("ConfigureValue.GetInt64() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfigureValue_GetBool(t *testing.T) { + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + want bool + wantErr bool + }{ + {"normal", fields{"ldap_verify_cert", "true"}, true, false}, + } + MetaData.InitMetaDataFromArray(testingMetaDataArray) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + got := c.GetBool() + if got != tt.want { + t.Errorf("ConfigureValue.GetBool() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfigureValue_GetStringToStringMap(t *testing.T) { + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + want map[string]string + wantErr bool + }{ + {"normal", fields{"sample_map_setting", `{ "value1":"abc","value2":"def" }`}, map[string]string{"value1": "abc", "value2": "def"}, false}, + } + MetaData.InitMetaDataFromArray(testingMetaDataArray) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + got := c.GetStringToStringMap() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ConfigureValue.GetStringToStringMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfigureValue_GetMap(t *testing.T) { + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + want map[string]interface{} + wantErr bool + }{ + {"normal", fields{"sample_map_setting", `{ "value1":"abc","value2":"def" }`}, map[string]interface{}{"value1": "abc", "value2": "def"}, false}, + } + MetaData.InitMetaDataFromArray(testingMetaDataArray) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + got := c.GetMap() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ConfigureValue.GetMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func LDAPScopeValidateFunc(key, value string) error { + if value == "1" || value == "2" || value == "3" { + return nil + } + return errors.New("The value should between 1, 2, 3") +} + +func TestConfigureValue_Validate(t *testing.T) { + type fields struct { + Key string + Value string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + {"out of scope", fields{"ldap_search_scope", "4"}, true}, + {"normal", fields{"ldap_search_scope", "3"}, false}, + } + + MetaData.InitMetaDataFromArray(testingMetaDataArray) + item, err := MetaData.GetConfigMetaData("ldap_search_scope") + if err != nil { + t.Errorf("Error occurred when GetConfigMetaData: %v", err) + } + item.Validator = LDAPScopeValidateFunc + MetaData.writeMap("ldap_search_scope", item) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + if err := c.Validate(); (err != nil) != tt.wantErr { + t.Errorf("ConfigureValue.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestConfigureValue_Set(t *testing.T) { + type fields struct { + Key string + Value string + } + type args struct { + key string + value string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + {"normal", fields{"", ""}, args{"ldap_search_scope", "4"}, true}, + {"normal", fields{"", ""}, args{"ldap_search_scope", "3"}, false}, + } + MetaData.InitMetaDataFromArray(testingMetaDataArray) + + item, err := MetaData.GetConfigMetaData("ldap_search_scope") + if err != nil { + t.Errorf("Error occurred when GetConfigMetaData: %v", err) + } + item.Validator = LDAPScopeValidateFunc + MetaData.writeMap("ldap_search_scope", item) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ConfigureValue{ + Key: tt.fields.Key, + Value: tt.fields.Value, + } + if err := c.Set(tt.args.key, tt.args.value); (err != nil) != tt.wantErr { + t.Errorf("ConfigureValue.Set() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/src/common/config/encrypt/encrypt.go b/src/common/config/encrypt/encrypt.go new file mode 100644 index 000000000000..d76bae0cf899 --- /dev/null +++ b/src/common/config/encrypt/encrypt.go @@ -0,0 +1,84 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package encrypt + +import ( + "os" + "sync" + + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" +) + +var ( + defaultKeyPath = "/etc/adminserver/key" +) + +// Encryptor encrypts or decrypts a strings +type Encryptor interface { + // Encrypt encrypts plaintext + Encrypt(string) (string, error) + // Decrypt decrypts ciphertext + Decrypt(string) (string, error) +} + +// AESEncryptor uses AES to encrypt or decrypt string +type AESEncryptor struct { + keyProvider KeyProvider + keyParams map[string]interface{} +} + +// NewAESEncryptor returns an instance of an AESEncryptor +func NewAESEncryptor(keyProvider KeyProvider, + keyParams map[string]interface{}) Encryptor { + return &AESEncryptor{ + keyProvider: keyProvider, + } +} + +var instance Encryptor +var once sync.Once + +// GetInstance ... +func GetInstance() Encryptor { + once.Do(func() { + kp := os.Getenv("KEY_PATH") + if len(kp) == 0 { + kp = defaultKeyPath + } + log.Infof("The path of key used by key provider: %s", kp) + instance = NewAESEncryptor(NewFileKeyProvider(kp), nil) + + }) + return instance +} + +// Encrypt ... +func (a *AESEncryptor) Encrypt(plaintext string) (string, error) { + key, err := a.keyProvider.Get(a.keyParams) + if err != nil { + return "", err + } + return utils.ReversibleEncrypt(plaintext, key) +} + +// Decrypt ... +func (a *AESEncryptor) Decrypt(ciphertext string) (string, error) { + key, err := a.keyProvider.Get(a.keyParams) + if err != nil { + return "", err + } + return utils.ReversibleDecrypt(ciphertext, key) +} diff --git a/src/adminserver/systemcfg/encrypt/encrypt_test.go b/src/common/config/encrypt/encrypt_test.go similarity index 94% rename from src/adminserver/systemcfg/encrypt/encrypt_test.go rename to src/common/config/encrypt/encrypt_test.go index 01f67d183a22..2089d230658a 100644 --- a/src/adminserver/systemcfg/encrypt/encrypt_test.go +++ b/src/common/config/encrypt/encrypt_test.go @@ -18,7 +18,6 @@ import ( "errors" "testing" - comcfg "github.com/goharbor/harbor/src/common/config" "github.com/stretchr/testify/assert" ) @@ -35,7 +34,7 @@ func (f *fakeKeyProvider) Get(params map[string]interface{}) ( func TestEncrypt(t *testing.T) { cases := []struct { plaintext string - keyProvider comcfg.KeyProvider + keyProvider KeyProvider err bool }{ {"", &fakeKeyProvider{"", errors.New("error")}, true}, @@ -72,7 +71,7 @@ func TestDecrypt(t *testing.T) { cases := []struct { ciphertext string - keyProvider comcfg.KeyProvider + keyProvider KeyProvider err bool }{ {"", &fakeKeyProvider{"", errors.New("error")}, true}, diff --git a/src/common/config/keyprovider.go b/src/common/config/encrypt/keyprovider.go similarity index 98% rename from src/common/config/keyprovider.go rename to src/common/config/encrypt/keyprovider.go index eac95612d235..0f6c6ced7df9 100644 --- a/src/common/config/keyprovider.go +++ b/src/common/config/encrypt/keyprovider.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package config +package encrypt import ( "io/ioutil" diff --git a/src/common/config/keyprovider_test.go b/src/common/config/encrypt/keyprovider_test.go similarity index 98% rename from src/common/config/keyprovider_test.go rename to src/common/config/encrypt/keyprovider_test.go index 48127d903796..8ac93962bb2b 100644 --- a/src/common/config/keyprovider_test.go +++ b/src/common/config/encrypt/keyprovider_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package config +package encrypt import ( "io/ioutil" diff --git a/src/common/const.go b/src/common/const.go index 4cb2d1c840b0..9d7f6cebaac5 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -201,4 +201,60 @@ var ( LDAPSearchPwd, UAAClientSecret, } + + // test functions under package core/config + TestServerDefaultConfig = map[string]interface{}{ + ExtEndpoint: "https://host01.com", + AUTHMode: DBAuth, + DatabaseType: "postgresql", + PostGreSQLHOST: "127.0.0.1", + PostGreSQLPort: 5432, + PostGreSQLUsername: "postgres", + PostGreSQLPassword: "root123", + PostGreSQLDatabase: "registry", + SelfRegistration: true, + LDAPURL: "ldap://127.0.0.1", + LDAPSearchDN: "uid=searchuser,ou=people,dc=mydomain,dc=com", + LDAPSearchPwd: "password", + LDAPBaseDN: "ou=people,dc=mydomain,dc=com", + LDAPUID: "uid", + LDAPFilter: "", + LDAPScope: 3, + LDAPTimeout: 30, + LDAPGroupBaseDN: "dc=example,dc=com", + LDAPGroupSearchFilter: "objectClass=groupOfNames", + LDAPGroupSearchScope: 2, + LDAPGroupAttributeName: "cn", + TokenServiceURL: "http://core:8080/service/token", + RegistryURL: "http://registry:5000", + EmailHost: "127.0.0.1", + EmailPort: 25, + EmailUsername: "user01", + EmailPassword: "password", + EmailFrom: "from", + EmailSSL: true, + EmailInsecure: false, + EmailIdentity: "", + ProjectCreationRestriction: ProCrtRestrAdmOnly, + MaxJobWorkers: 3, + TokenExpiration: 30, + CfgExpiration: 5, + AdminInitialPassword: "password", + AdmiralEndpoint: "", + WithNotary: false, + WithClair: false, + ClairDBUsername: "postgres", + ClairDBHost: "postgresql", + ClairDB: "postgres", + ClairDBPort: 5432, + ClairDBPassword: "root123", + UAAClientID: "testid", + UAAClientSecret: "testsecret", + UAAEndpoint: "10.192.168.5", + UAAVerifyCert: false, + CoreURL: "http://core:8080/", + JobServiceURL: "http://jobservice:8080", + ReadOnly: false, + NotaryURL: "http://notary-server:4443", + } ) diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 4de0f8648aea..921b2a1d6e6e 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -169,3 +169,16 @@ func Escape(str string) string { str = strings.Replace(str, `_`, `\_`, -1) return str } + +// PrepareDatabase ... create table or upgrade +func PrepareDatabase(database *models.Database) { + if err := InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + if err := UpgradeSchema(database); err != nil { + log.Fatalf("failed to upgrade database schema: %v", err) + } + if err := CheckSchemaVersion(); err != nil { + log.Fatalf("failed to check database schema version: %v", err) + } +} diff --git a/src/common/security/local/context_test.go b/src/common/security/local/context_test.go index 976237fd6ee6..f1ac2d37048e 100644 --- a/src/common/security/local/context_test.go +++ b/src/common/security/local/context_test.go @@ -16,10 +16,10 @@ package local import ( "os" - "strconv" "testing" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/models" @@ -53,45 +53,8 @@ var ( ) func TestMain(m *testing.M) { - dbHost := os.Getenv("POSTGRESQL_HOST") - if len(dbHost) == 0 { - log.Fatalf("environment variable POSTGRES_HOST is not set") - } - dbUser := os.Getenv("POSTGRESQL_USR") - if len(dbUser) == 0 { - log.Fatalf("environment variable POSTGRES_USR is not set") - } - dbPortStr := os.Getenv("POSTGRESQL_PORT") - if len(dbPortStr) == 0 { - log.Fatalf("environment variable POSTGRES_PORT is not set") - } - dbPort, err := strconv.Atoi(dbPortStr) - if err != nil { - log.Fatalf("invalid POSTGRESQL_PORT: %v", err) - } - dbPassword := os.Getenv("POSTGRESQL_PWD") - dbDatabase := os.Getenv("POSTGRESQL_DATABASE") - if len(dbDatabase) == 0 { - log.Fatalf("environment variable POSTGRESQL_DATABASE is not set") - } - - database := &models.Database{ - Type: "postgresql", - PostGreSQL: &models.PostGreSQL{ - Host: dbHost, - Port: dbPort, - Username: dbUser, - Password: dbPassword, - Database: dbDatabase, - }, - } - - log.Infof("POSTGRES_HOST: %s, POSTGRES_USR: %s, POSTGRES_PORT: %d, POSTGRES_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) - - if err := dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } + db.InitDatabaseAndConfigure() // regiser users id, err := dao.Register(*projectAdminUser) diff --git a/src/common/utils/ldap/ldap_test.go b/src/common/utils/ldap/ldap_test.go index fd22a4b0fe1b..bcacd59e39ee 100644 --- a/src/common/utils/ldap/ldap_test.go +++ b/src/common/utils/ldap/ldap_test.go @@ -1,12 +1,13 @@ package ldap import ( + "fmt" "os" "reflect" "testing" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/test" @@ -30,9 +31,10 @@ var adminServerLdapTestConfig = map[string]interface{}{ common.LDAPBaseDN: "dc=example,dc=com", common.LDAPUID: "uid", common.LDAPFilter: "", - common.LDAPScope: 3, + common.LDAPScope: 2, common.LDAPTimeout: 30, common.CfgExpiration: 5, + common.LDAPVerifyCert: false, common.AdminInitialPassword: "password", } @@ -100,16 +102,11 @@ func TestMain(m *testing.M) { if err := uiConfig.Init(); err != nil { log.Fatalf("failed to initialize configurations: %v", err) } - - database, err := uiConfig.Database() - if err != nil { - log.Fatalf("failed to get database configuration: %v", err) - } - - if err := dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } - + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(adminServerLdapTestConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) os.Exit(m.Run()) } diff --git a/src/common/utils/notary/helper_test.go b/src/common/utils/notary/helper_test.go index 6bfec989e581..ab84c6cd6eb3 100644 --- a/src/common/utils/notary/helper_test.go +++ b/src/common/utils/notary/helper_test.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" notarytest "github.com/goharbor/harbor/src/common/utils/notary/test" utilstest "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/core/config" @@ -53,6 +54,13 @@ func TestMain(m *testing.M) { if err := config.Init(); err != nil { panic(err) } + + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(defaultConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) + notaryCachePath = "/tmp/notary" result := m.Run() if result != 0 { diff --git a/src/core/api/api_test.go b/src/core/api/api_test.go index 8b7da6a75bdb..1f2a7c88eeba 100644 --- a/src/core/api/api_test.go +++ b/src/core/api/api_test.go @@ -203,6 +203,7 @@ func TestMain(m *testing.M) { if err := prepare(); err != nil { panic(err) } + fmt.Printf("Test env prepared") ret := m.Run() clean() diff --git a/src/core/api/configsetting.go b/src/core/api/configsetting.go new file mode 100644 index 000000000000..5644afe66f0a --- /dev/null +++ b/src/core/api/configsetting.go @@ -0,0 +1,46 @@ +package api + +import ( + "net/http" + + "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/config/client/db" + "github.com/goharbor/harbor/src/common/utils/log" +) + +// ConfigSettingAPI ... +type ConfigSettingAPI struct { + BaseController + ConfigClient config.Client +} + +// Prepare validates the user +func (c *ConfigSettingAPI) Prepare() { + c.BaseController.Prepare() + if !c.SecurityCtx.IsAuthenticated() { + c.HandleUnauthorized() + return + } + if !c.SecurityCtx.IsSysAdmin() && !c.SecurityCtx.IsSolutionUser() { + c.HandleForbidden(c.SecurityCtx.GetUsername()) + return + } + c.ConfigClient = db.NewDBConfigureStore() +} + +// GetConfigByGroup ... +func (c *ConfigSettingAPI) GetConfigByGroup() { + group := c.GetStringFromPath(":group") + if len(group) == 0 { + log.Error("failed to get group") + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + configList, err := c.ConfigClient.GetSettingByGroup(group) + log.Errorf("Found items %v", len(configList)) + if err != nil { + log.Error("failed to get setting by group %v", err) + } + c.Data["json"] = configList + c.ServeJSON() + return +} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index e082facdfcbd..a068e7e913ce 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -27,6 +27,8 @@ import ( "runtime" "strconv" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/job/test" "github.com/goharbor/harbor/src/common/models" @@ -78,6 +80,47 @@ type usrInfo struct { Passwd string } +var adminServerAPITestConfig = map[string]interface{}{ + common.ExtEndpoint: "host01.com", + common.AUTHMode: "db_auth", + common.DatabaseType: "postgresql", + common.PostGreSQLHOST: "127.0.0.1", + common.PostGreSQLPort: 5432, + common.PostGreSQLUsername: "postgres", + common.PostGreSQLPassword: "root123", + common.PostGreSQLDatabase: "registry", + // config.SelfRegistration: true, + common.LDAPURL: "ldap://127.0.0.1", + common.LDAPSearchDN: "cn=admin,dc=example,dc=com", + common.LDAPSearchPwd: "admin", + common.LDAPBaseDN: "dc=example,dc=com", + common.LDAPUID: "uid", + common.LDAPFilter: "", + common.LDAPScope: 2, + common.LDAPTimeout: 30, + // config.TokenServiceURL: "", + // config.RegistryURL: "", + // config.EmailHost: "", + // config.EmailPort: 25, + // config.EmailUsername: "", + // config.EmailPassword: "password", + // config.EmailFrom: "from", + // config.EmailSSL: true, + // config.EmailIdentity: "", + // config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, + // config.VerifyRemoteCert: false, + // config.MaxJobWorkers: 3, + // config.TokenExpiration: 30, + common.CfgExpiration: 5, + // config.JobLogDir: "/var/log/jobs", + common.AdminInitialPassword: "password", + common.LDAPGroupSearchFilter: "objectclass=groupOfNames", + common.LDAPGroupBaseDN: "dc=example,dc=com", + common.LDAPGroupAttributeName: "cn", + common.LDAPGroupSearchScope: 2, + common.LdapGroupAdminDn: "cn=harbor_users,ou=groups,dc=example,dc=com", +} + func init() { ldapConfig := models.LdapConf{ LdapURL: "ldap://127.0.0.1:389", @@ -101,11 +144,11 @@ func init() { if err := config.Init(); err != nil { log.Fatalf("failed to initialize configurations: %v", err) } - database, err := config.Database() - if err != nil { - log.Fatalf("failed to get database configurations: %v", err) - } - dao.InitDatabase(database) + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(adminServerAPITestConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) _, file, _, _ := runtime.Caller(0) dir := filepath.Dir(file) dir = filepath.Join(dir, "..") diff --git a/src/core/auth/db/db_test.go b/src/core/auth/db/db_test.go index c1a074192769..3b6a3504ac8d 100644 --- a/src/core/auth/db/db_test.go +++ b/src/core/auth/db/db_test.go @@ -14,11 +14,13 @@ package db import ( + "fmt" "log" "os" "testing" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/utils/test" @@ -91,14 +93,11 @@ func TestMain(m *testing.M) { log.Fatalf("failed to initialize configurations: %v", err) } - database, err := coreConfig.Database() - if err != nil { - log.Fatalf("failed to get database configuration: %v", err) - } - - if err := dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(adminServerTestConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) } func TestSearchUser(t *testing.T) { diff --git a/src/core/auth/ldap/ldap_test.go b/src/core/auth/ldap/ldap_test.go index 27e84e203e67..69f3a07879eb 100644 --- a/src/core/auth/ldap/ldap_test.go +++ b/src/core/auth/ldap/ldap_test.go @@ -14,6 +14,8 @@ package ldap import ( + "fmt" + "github.com/stretchr/testify/assert" // "fmt" // "strings" @@ -21,6 +23,7 @@ import ( "testing" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/dao/project" "github.com/goharbor/harbor/src/common/models" @@ -100,14 +103,11 @@ func TestMain(m *testing.M) { log.Fatalf("failed to initialize configurations: %v", err) } - database, err := coreConfig.Database() - if err != nil { - log.Fatalf("failed to get database configuration: %v", err) - } - - if err := dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(adminServerLdapTestConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) // Extract to test utils initSqls := []string{ diff --git a/src/core/config/config.go b/src/core/config/config.go index ccaf2c18fd4a..5781d8f20880 100644 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -29,6 +29,8 @@ import ( "github.com/goharbor/harbor/src/adminserver/client" "github.com/goharbor/harbor/src/common" comcfg "github.com/goharbor/harbor/src/common/config" + "github.com/goharbor/harbor/src/common/config/client/db" + "github.com/goharbor/harbor/src/common/config/encrypt" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/secret" "github.com/goharbor/harbor/src/common/utils" @@ -52,8 +54,9 @@ var ( AdminserverClient client.Client // GlobalProjectMgr is initialized based on the deploy mode GlobalProjectMgr promgr.ProjectManager - mg *comcfg.Manager - keyProvider comcfg.KeyProvider + // mg *comcfg.Manager + mg comcfg.ManagerInterface + keyProvider encrypt.KeyProvider // AdmiralClient is initialized only under integration deploy mode // and can be passed to project manager as a parameter AdmiralClient *http.Client @@ -67,32 +70,21 @@ var ( func Init() error { // init key provider initKeyProvider() - adminServerURL := os.Getenv("ADMINSERVER_URL") - if len(adminServerURL) == 0 { - adminServerURL = common.DefaultAdminserverEndpoint - } - - return InitByURL(adminServerURL) - + // adminServerURL := os.Getenv("ADMINSERVER_URL") + // if len(adminServerURL) == 0 { + // adminServerURL = common.DefaultAdminserverEndpoint + // } + return InitDBConfigManager() } -// InitByURL Init configurations with given url -func InitByURL(adminServerURL string) error { - log.Infof("initializing client for adminserver %s ...", adminServerURL) - cfg := &client.Config{ - Secret: CoreSecret(), - } - AdminserverClient = client.NewClient(adminServerURL, cfg) - if err := AdminserverClient.Ping(); err != nil { - return fmt.Errorf("failed to ping adminserver: %v", err) - } - - mg = comcfg.NewManager(AdminserverClient, true) +// InitDBConfigManager ... +func InitDBConfigManager() error { + mg = db.NewCoreConfigManager() - if err := Load(); err != nil { + _, err := mg.Load() + if err != nil { return err } - // init secret store initSecretStore() @@ -102,7 +94,7 @@ func InitByURL(adminServerURL string) error { return err } - return nil + return err } func initKeyProvider() { @@ -112,7 +104,7 @@ func initKeyProvider() { } log.Infof("key path: %s", path) - keyProvider = comcfg.NewFileKeyProvider(path) + keyProvider = encrypt.NewFileKeyProvider(path) } func initSecretStore() { @@ -214,8 +206,8 @@ func LDAPConf() (*models.LdapConf, error) { ldapConf.LdapBaseDn = utils.SafeCastString(cfg[common.LDAPBaseDN]) ldapConf.LdapUID = utils.SafeCastString(cfg[common.LDAPUID]) ldapConf.LdapFilter = utils.SafeCastString(cfg[common.LDAPFilter]) - ldapConf.LdapScope = int(utils.SafeCastFloat64(cfg[common.LDAPScope])) - ldapConf.LdapConnectionTimeout = int(utils.SafeCastFloat64(cfg[common.LDAPTimeout])) + ldapConf.LdapScope = utils.SafeCastInt(cfg[common.LDAPScope]) + ldapConf.LdapConnectionTimeout = utils.SafeCastInt(cfg[common.LDAPTimeout]) if cfg[common.LDAPVerifyCert] != nil { ldapConf.LdapVerifyCert = cfg[common.LDAPVerifyCert].(bool) } else { @@ -316,7 +308,7 @@ func RegistryURL() (string, error) { func InternalJobServiceURL() string { cfg, err := mg.Get() if err != nil { - log.Warningf("Failed to Get job service URL from backend, error: %v, will return default value.") + log.Warningf("Failed to Get job service URL from backend, error: %v, will return default value.", err) return common.DefaultJobserviceEndpoint } @@ -330,7 +322,7 @@ func InternalJobServiceURL() string { func InternalCoreURL() string { cfg, err := mg.Get() if err != nil { - log.Warningf("Failed to Get job service Core URL from backend, error: %v, will return default value.") + log.Warningf("Failed to Get job service Core URL from backend, error: %v, will return default value.", err) return common.DefaultCoreEndpoint } return strings.TrimSuffix(utils.SafeCastString(cfg[common.CoreURL]), "/") @@ -347,7 +339,7 @@ func InternalTokenServiceEndpoint() string { func InternalNotaryEndpoint() string { cfg, err := mg.Get() if err != nil { - log.Warningf("Failed to get Notary endpoint from backend, error: %v, will use default value.") + log.Warningf("Failed to get Notary endpoint from backend, error: %v, will use default value.", err) return common.DefaultNotaryEndpoint } if cfg[common.NotaryURL] == nil { @@ -401,12 +393,14 @@ func Database() (*models.Database, error) { return nil, err } + log.Infof("database connections = %+v", cfg) + database := &models.Database{} - database.Type = cfg[common.DatabaseType].(string) + database.Type = utils.SafeCastString(cfg[common.DatabaseType]) postgresql := &models.PostGreSQL{} postgresql.Host = utils.SafeCastString(cfg[common.PostGreSQLHOST]) - postgresql.Port = int(utils.SafeCastFloat64(cfg[common.PostGreSQLPort])) + postgresql.Port = utils.SafeCastInt(cfg[common.PostGreSQLPort]) postgresql.Username = utils.SafeCastString(cfg[common.PostGreSQLUsername]) postgresql.Password = utils.SafeCastString(cfg[common.PostGreSQLPassword]) postgresql.Database = utils.SafeCastString(cfg[common.PostGreSQLDatabase]) @@ -468,7 +462,7 @@ func ClairDB() (*models.PostGreSQL, error) { } clairDB := &models.PostGreSQL{} clairDB.Host = utils.SafeCastString(cfg[common.ClairDBHost]) - clairDB.Port = int(utils.SafeCastFloat64(cfg[common.ClairDBPort])) + clairDB.Port = utils.SafeCastInt(cfg[common.ClairDBPort]) clairDB.Username = utils.SafeCastString(cfg[common.ClairDBUsername]) clairDB.Password = utils.SafeCastString(cfg[common.ClairDBPassword]) clairDB.Database = utils.SafeCastString(cfg[common.ClairDB]) diff --git a/src/core/config/config_test.go b/src/core/config/config_test.go index 2e0e0dd6539f..e250dc1e92e2 100644 --- a/src/core/config/config_test.go +++ b/src/core/config/config_test.go @@ -20,11 +20,11 @@ import ( "testing" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/utils/test" "github.com/stretchr/testify/assert" ) -// test functions under package core/config func TestConfig(t *testing.T) { defaultCACertPath = path.Join(currPath(), "test", "ca.crt") @@ -54,6 +54,16 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to set env %s: %v", "KEY_PATH", err) } + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(common.TestServerDefaultConfig) + admiralSettingMap := map[string]interface{}{ + common.AdmiralEndpoint: "http://www.vmware.com", + common.JobServiceURL: "http://myjob:8888", + common.CoreURL: "http://myui:8888", + } + cfgManager.Upload(admiralSettingMap) + if err := Init(); err != nil { t.Fatalf("failed to initialize configurations: %v", err) } @@ -193,7 +203,6 @@ func TestConfig(t *testing.T) { if err != nil { t.Fatalf("failed to get UAA setting, error: %v", err) } - if us.ClientID != "testid" || us.ClientSecret != "testsecret" || us.Endpoint != "10.192.168.5" || us.VerifyCert { t.Errorf("Unexpected UAA setting: %+v", *us) } diff --git a/src/core/controllers/controllers_test.go b/src/core/controllers/controllers_test.go index 85a7596a25e2..83a3d69958e7 100644 --- a/src/core/controllers/controllers_test.go +++ b/src/core/controllers/controllers_test.go @@ -27,9 +27,10 @@ import ( "github.com/astaxie/beego" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/test" + _ "github.com/goharbor/harbor/src/core/auth/db" + _ "github.com/goharbor/harbor/src/core/auth/ldap" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/proxy" "github.com/stretchr/testify/assert" @@ -67,6 +68,14 @@ func init() { } func TestMain(m *testing.M) { + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(common.TestServerDefaultConfig) + cfg, err := cfgManager.Get() + if err != nil { + fmt.Printf("Error occurred when get config: %v", err) + } + fmt.Printf("config settings,cfg:%v\n", cfg) rc := m.Run() if rc != 0 { @@ -84,23 +93,14 @@ func TestUserResettable(t *testing.T) { common.CfgExpiration: 5, common.TokenExpiration: 30, } - + cfgManager := db.NewCoreConfigManager() LDAPAuthConfig := map[string]interface{}{ common.AUTHMode: common.LDAPAuth, common.CfgExpiration: 5, common.TokenExpiration: 30, } - DBAuthAdminsvr, err := test.NewAdminserver(DBAuthConfig) - if err != nil { - panic(err) - } - LDAPAuthAdminsvr, err := test.NewAdminserver(LDAPAuthConfig) - if err != nil { - panic(err) - } - defer DBAuthAdminsvr.Close() - defer LDAPAuthAdminsvr.Close() - if err := config.InitByURL(LDAPAuthAdminsvr.URL); err != nil { + cfgManager.Upload(LDAPAuthConfig) + if err := config.Init(); err != nil { panic(err) } u1 := &models.User{ @@ -114,28 +114,33 @@ func TestUserResettable(t *testing.T) { Email: "jack@test.com", } assert.False(isUserResetable(u1)) + cfgManager.Upload(DBAuthConfig) assert.True(isUserResetable(u2)) - if err := config.InitByURL(DBAuthAdminsvr.URL); err != nil { - panic(err) - } - assert.True(isUserResetable(u1)) + } // TestMain is a sample to run an endpoint test func TestAll(t *testing.T) { + cfgManager := db.NewCoreConfigManager() + TestConfig := map[string]interface{}{ + common.RegistryURL: "http://" + os.Getenv("REGISTRY_URL"), + } + cfgManager.Upload(TestConfig) + if err := config.Init(); err != nil { panic(err) } if err := proxy.Init(); err != nil { panic(err) } - database, err := config.Database() - if err != nil { - panic(err) - } - if err := dao.InitDatabase(database); err != nil { - panic(err) - } + + // database, err := config.Database() + // if err != nil { + // panic(err) + // } + // if err := dao.InitDatabase(database); err != nil { + // panic(err) + // } assert := assert.New(t) diff --git a/src/core/filter/readonly_test.go b/src/core/filter/readonly_test.go index fdc70b50a990..a4249cfe3145 100644 --- a/src/core/filter/readonly_test.go +++ b/src/core/filter/readonly_test.go @@ -17,11 +17,10 @@ package filter import ( "net/http" "net/http/httptest" - "os" "testing" "github.com/goharbor/harbor/src/common" - utilstest "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" ) @@ -41,14 +40,19 @@ func TestReadonlyFilter(t *testing.T) { common.PostGreSQLUsername: "postgres", common.ReadOnly: true, } - adminServer, err := utilstest.NewAdminserver(defaultConfig) - if err != nil { - panic(err) - } - defer adminServer.Close() - if err := os.Setenv("ADMINSERVER_URL", adminServer.URL); err != nil { - panic(err) - } + + // adminServer, err := utilstest.NewAdminserver(defaultConfig) + // if err != nil { + // panic(err) + // } + // defer adminServer.Close() + // if err := os.Setenv("ADMINSERVER_URL", adminServer.URL); err != nil { + // panic(err) + // } + // db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(defaultConfig) + if err := config.Init(); err != nil { panic(err) } diff --git a/src/core/filter/security.go b/src/core/filter/security.go index 78ec48ba3733..95574d5daec0 100644 --- a/src/core/filter/security.go +++ b/src/core/filter/security.go @@ -240,7 +240,6 @@ func (s *sessionReqCtxModifier) Modify(ctx *beegoctx.Context) bool { log.Info("can not get user information from session") return false } - log.Debug("Getting user %+v", user) log.Debug("using local database project manager") pm := config.GlobalProjectMgr log.Debug("creating local database security context...") diff --git a/src/core/filter/security_test.go b/src/core/filter/security_test.go index 3512c61a2561..3d5f1daa2ab7 100644 --- a/src/core/filter/security_test.go +++ b/src/core/filter/security_test.go @@ -16,6 +16,7 @@ package filter import ( "context" + "fmt" "log" "net/http" "net/http/httptest" @@ -28,7 +29,8 @@ import ( "github.com/astaxie/beego" beegoctx "github.com/astaxie/beego/context" "github.com/astaxie/beego/session" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/models" commonsecret "github.com/goharbor/harbor/src/common/secret" "github.com/goharbor/harbor/src/common/security" @@ -42,6 +44,47 @@ import ( "github.com/stretchr/testify/assert" ) +var adminServerSecurityTestConfig = map[string]interface{}{ + common.ExtEndpoint: "host01.com", + common.AUTHMode: "db_auth", + common.DatabaseType: "postgresql", + common.PostGreSQLHOST: "127.0.0.1", + common.PostGreSQLPort: 5432, + common.PostGreSQLUsername: "postgres", + common.PostGreSQLPassword: "root123", + common.PostGreSQLDatabase: "registry", + // config.SelfRegistration: true, + common.LDAPURL: "ldap://127.0.0.1", + common.LDAPSearchDN: "cn=admin,dc=example,dc=com", + common.LDAPSearchPwd: "admin", + common.LDAPBaseDN: "dc=example,dc=com", + common.LDAPUID: "uid", + common.LDAPFilter: "", + common.LDAPScope: 2, + common.LDAPTimeout: 30, + // config.TokenServiceURL: "", + // config.RegistryURL: "", + // config.EmailHost: "", + // config.EmailPort: 25, + // config.EmailUsername: "", + // config.EmailPassword: "password", + // config.EmailFrom: "from", + // config.EmailSSL: true, + // config.EmailIdentity: "", + // config.ProjectCreationRestriction: config.ProCrtRestrAdmOnly, + // config.VerifyRemoteCert: false, + // config.MaxJobWorkers: 3, + // config.TokenExpiration: 30, + common.CfgExpiration: 5, + // config.JobLogDir: "/var/log/jobs", + common.AdminInitialPassword: "password", + common.LDAPGroupSearchFilter: "objectclass=groupOfNames", + common.LDAPGroupBaseDN: "dc=example,dc=com", + common.LDAPGroupAttributeName: "cn", + common.LDAPGroupSearchScope: 2, + common.LdapGroupAdminDn: "cn=harbor_users,ou=groups,dc=example,dc=com", +} + func TestMain(m *testing.M) { // initialize beego session manager conf := &session.ManagerConfig{ @@ -63,13 +106,12 @@ func TestMain(m *testing.M) { if err := config.Init(); err != nil { log.Fatalf("failed to initialize configurations: %v", err) } - database, err := config.Database() - if err != nil { - log.Fatalf("failed to get database configurations: %v", err) - } - if err = dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } + + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(adminServerSecurityTestConfig) + cfg, err := cfgManager.Get() + fmt.Printf("config settings,cfg:%v\n", cfg) Init() diff --git a/src/core/main.go b/src/core/main.go index d9eba4bece83..f646e7dd7ea0 100644 --- a/src/core/main.go +++ b/src/core/main.go @@ -23,6 +23,7 @@ import ( "github.com/astaxie/beego" _ "github.com/astaxie/beego/session/redis" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" @@ -87,14 +88,7 @@ func main() { } log.Info("configurations initialization completed") token.InitCreators() - database, err := config.Database() - if err != nil { - log.Fatalf("failed to get database configuration: %v", err) - } - if err := dao.InitDatabase(database); err != nil { - log.Fatalf("failed to initialize database: %v", err) - } - + db.InitDatabaseAndConfigure() password, err := config.InitialAdminPassword() if err != nil { log.Fatalf("failed to get admin's initia password: %v", err) diff --git a/src/core/promgr/pmsdriver/local/local_test.go b/src/core/promgr/pmsdriver/local/local_test.go index ec985f3567ba..db028271c39e 100644 --- a/src/core/promgr/pmsdriver/local/local_test.go +++ b/src/core/promgr/pmsdriver/local/local_test.go @@ -18,14 +18,14 @@ import ( "os" "testing" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/models" errutil "github.com/goharbor/harbor/src/common/utils/error" - "github.com/goharbor/harbor/src/common/utils/test" "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { - test.InitDatabaseFromEnv() + db.InitDatabaseAndConfigure() os.Exit(m.Run()) } diff --git a/src/core/proxy/interceptor_test.go b/src/core/proxy/interceptor_test.go index 5f4794c24112..5507e84cb1a6 100644 --- a/src/core/proxy/interceptor_test.go +++ b/src/core/proxy/interceptor_test.go @@ -3,7 +3,7 @@ package proxy import ( "github.com/goharbor/harbor/src/adminserver/client" "github.com/goharbor/harbor/src/common" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/config/client/db" "github.com/goharbor/harbor/src/common/models" notarytest "github.com/goharbor/harbor/src/common/utils/notary/test" utilstest "github.com/goharbor/harbor/src/common/utils/test" @@ -134,13 +134,10 @@ func TestPMSPolicyChecker(t *testing.T) { if err := config.Init(); err != nil { panic(err) } - database, err := config.Database() - if err != nil { - panic(err) - } - if err := dao.InitDatabase(database); err != nil { - panic(err) - } + + db.InitDatabaseAndConfigure() + cfgManager := db.NewCoreConfigManager() + cfgManager.Upload(defaultConfigAdmiral) name := "project_for_test_get_sev_low" id, err := config.GlobalProjectMgr.Create(&models.Project{ diff --git a/src/core/proxy/proxy.go b/src/core/proxy/proxy.go index 930f9290db37..e061811bd51f 100644 --- a/src/core/proxy/proxy.go +++ b/src/core/proxy/proxy.go @@ -27,6 +27,7 @@ func Init(urls ...string) error { } if len(urls) == 0 { registryURL, err = config.RegistryURL() + fmt.Printf("registry URL:%v\n", registryURL) if err != nil { return err } diff --git a/src/core/router.go b/src/core/router.go index 2c629ba01a0b..a49d45ffb652 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -98,6 +98,9 @@ func initRouters() { beego.Router("/api/targets/ping", &api.TargetAPI{}, "post:Ping") beego.Router("/api/logs", &api.LogAPI{}) beego.Router("/api/configs", &api.ConfigAPI{}, "get:GetInternalConfig") + + beego.Router("/api/config/:group", &api.ConfigSettingAPI{}, "get:GetConfigByGroup") + beego.Router("/api/configurations", &api.ConfigAPI{}) beego.Router("/api/configurations/reset", &api.ConfigAPI{}, "post:Reset") beego.Router("/api/statistics", &api.StatisticAPI{}) diff --git a/src/core/utils/job.go b/src/core/utils/job.go index b157c49a1f1f..37fe12e4c19d 100644 --- a/src/core/utils/job.go +++ b/src/core/utils/job.go @@ -129,7 +129,7 @@ func triggerImageScan(repository, tag, digest string, client job.Client) error { } err = dao.SetScanJobUUID(id, uuid) if err != nil { - log.Warningf("Failed to set UUID for scan job, ID: %d, repository: %s, tag: %s", id, uuid, repository, tag) + log.Warningf("Failed to set UUID for scan job, ID: %d, uuid: %s, repository: %s, tag: %s", id, uuid, repository, tag) } return nil } diff --git a/src/jobservice/job/impl/context.go b/src/jobservice/job/impl/context.go index 127eb7d35001..7edf1fdf0b2e 100644 --- a/src/jobservice/job/impl/context.go +++ b/src/jobservice/job/impl/context.go @@ -22,8 +22,8 @@ import ( "reflect" "time" - "github.com/goharbor/harbor/src/adminserver/client" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/config/client/remote" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/jobservice/config" @@ -57,16 +57,15 @@ type Context struct { // other required information properties map[string]interface{} - // admin server client - adminClient client.Client + remoteClient remote.Client } // NewContext ... -func NewContext(sysCtx context.Context, adminClient client.Client) *Context { +func NewContext(sysCtx context.Context, remoteClient remote.Client) *Context { return &Context{ - sysContext: sysCtx, - adminClient: adminClient, - properties: make(map[string]interface{}), + sysContext: sysCtx, + remoteClient: remoteClient, + properties: make(map[string]interface{}), } } @@ -75,12 +74,11 @@ func (c *Context) Init() error { var ( counter = 0 err error - configs map[string]interface{} ) - + var db *models.Database for counter == 0 || err != nil { counter++ - configs, err = c.adminClient.GetCfgs() + db, err = c.remoteClient.GetDatabaseCfg() if err != nil { logger.Errorf("Job context initialization error: %s\n", err.Error()) if counter < maxRetryTimes { @@ -92,9 +90,6 @@ func (c *Context) Init() error { } } } - - db := getDBFromConfig(configs) - return dao.InitDatabase(db) } @@ -102,9 +97,9 @@ func (c *Context) Init() error { // This func will build the job execution context before running func (c *Context) Build(dep env.JobData) (env.JobContext, error) { jContext := &Context{ - sysContext: c.sysContext, - adminClient: c.adminClient, - properties: make(map[string]interface{}), + sysContext: c.sysContext, + remoteClient: c.remoteClient, + properties: make(map[string]interface{}), } // Copy properties @@ -115,7 +110,7 @@ func (c *Context) Build(dep env.JobData) (env.JobContext, error) { } // Refresh admin server properties - props, err := c.adminClient.GetCfgs() + props, err := c.remoteClient.GetCfgs() if err != nil { return nil, err } diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 0c9ed923c325..ee364c69a81b 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -20,7 +20,7 @@ import ( "flag" "fmt" - "github.com/goharbor/harbor/src/adminserver/client" + "github.com/goharbor/harbor/src/common/config/client/remote" "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/job/impl" @@ -60,9 +60,11 @@ func main() { if utils.IsEmptyStr(secret) { return nil, errors.New("empty auth secret") } - - adminClient := client.NewClient(config.GetAdminServerEndpoint(), &client.Config{Secret: secret}) - jobCtx := impl.NewContext(ctx.SystemContext, adminClient) + remoteClient, err := remote.NewRemoteConfigDriver(config.GetAdminServerEndpoint(), &remote.Config{Secret: secret}) + if err != nil { + return nil, err + } + jobCtx := impl.NewContext(ctx.SystemContext, remoteClient) if err := jobCtx.Init(); err != nil { return nil, err diff --git a/src/replication/trigger/manager.go b/src/replication/trigger/manager.go index 40bed8491db9..8449e7354c73 100644 --- a/src/replication/trigger/manager.go +++ b/src/replication/trigger/manager.go @@ -19,9 +19,7 @@ type Manager struct { // NewManager is the constructor of trigger manager. // capacity is the max number of trigger references manager can keep in memory func NewManager(capacity int) *Manager { - return &Manager{ - // cache: NewCache(capacity), - } + return &Manager{} } /* diff --git a/tests/configharbor.py b/tests/configharbor.py new file mode 100644 index 000000000000..2aaeb52ff4b8 --- /dev/null +++ b/tests/configharbor.py @@ -0,0 +1,52 @@ +#!/usr/bin/python + +import argparse +import requests +import urllib3 +import json +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("-H", "--host", help="The Harbor server need to config") +parser.add_argument("-u", "--user", default="admin", help="The Harbor username") +parser.add_argument("-p", "--password", default="Harbor12345", help="The Harbor password") +parser.add_argument("-c", "--config", nargs='+', help="The configure settings =, it can take more than one configures") +args = parser.parse_args() +reqJson = {} +for item in args.config : + configs = item.split("=", 1) + key = configs[0].strip() + value = configs[1].strip() + if value.lower() in ['true', 'yes', '1'] : + reqJson[key] = True + elif value.lower() in ['false', 'no', '0'] : + reqJson[key] = False + elif value.isdigit() : + reqJson[key] = int(value) + else: + reqJson[key] = value + +#urllib3.disable_warnings() +# Sample Basic Auth Url with login values as username and password +url = "https://"+args.host+"/api/configurations" +user = args.user +passwd = args.password + + +# Make a request to the endpoint using the correct auth values +auth_values = (user, passwd) +session = requests.Session() +session.verify = False +data = json.dumps(reqJson) +headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} +response = session.put(url, auth=auth_values, data=data, headers=headers) + +# Convert JSON to dict and print +if response.status_code == 200 : + print("Configure setting success") + print("values:"+data) + sys.exit(0) +else: + print("Failed with http return code:"+ str(response.status_code)) + sys.exit(1) + diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index 566ba0637f55..a13c22b79166 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -19,17 +19,6 @@ services: - ./common/config/db/env ports: - 5432:5432 - adminserver: - image: goharbor/harbor-adminserver:__version__ - env_file: - - ./common/config/adminserver/env - restart: always - volumes: - - /data/config/:/etc/adminserver/config/ - - /data/secretkey:/etc/adminserver/key - - /data/:/data/ - ports: - - 8888:8080 redis: image: goharbor/redis-photon:4.0 restart: always diff --git a/tests/hostcfg.sh b/tests/hostcfg.sh index aeb25fdc492e..f52cfd21c4cf 100755 --- a/tests/hostcfg.sh +++ b/tests/hostcfg.sh @@ -5,12 +5,12 @@ PROTOCOL='https' #echo $IP sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.cfg sudo sed "s/^ui_url_protocol = .*/ui_url_protocol = $PROTOCOL/g" -i make/harbor.cfg - +pip list if [ "$1" = 'LDAP' ]; then - sudo sed "s/db_auth/ldap_auth/" -i make/harbor.cfg - sudo sed "s/ldaps:\/\/ldap.mydomain.com/ldap:\/\/$IP/g" -i make/harbor.cfg - sudo sed "s/#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com/ldap_searchdn = cn=admin,dc=example,dc=com/" -i make/harbor.cfg - sudo sed "s/#ldap_search_pwd = password/ldap_search_pwd = admin/" -i make/harbor.cfg - sudo sed "s/ldap_basedn = ou=people,dc=mydomain,dc=com/ldap_basedn = dc=example,dc=com/" -i make/harbor.cfg - sudo sed "s/ldap_uid = uid/ldap_uid = cn/" -i make/harbor.cfg + python ./tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c db_auth=ldap_auth \ + ldap_url=ldap://$IP \ + ldap_search_dn=cn=admin,dc=example,dc=com \ + ldap_search_password=admin \ + ldap_search_dn=dc=example,dc=com \ + ldap_uid=cn fi \ No newline at end of file diff --git a/tests/testprepare.sh b/tests/testprepare.sh index 54ec9ec9a7a0..45756e33aa97 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -13,8 +13,4 @@ else IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` fi echo "server ip is "$IP -sed -i -r "s/POSTGRESQL_HOST=postgresql/POSTGRESQL_HOST=$IP/" make/common/config/adminserver/env -sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://$IP:5000|" make/common/config/adminserver/env -sed -i -r "s/CORE_SECRET=.*/CORE_SECRET=$CORE_SECRET/" make/common/config/adminserver/env - chmod 777 /data/ diff --git a/tests/travis/ut_install.sh b/tests/travis/ut_install.sh index d0ec473c8d67..1bf39d2756ad 100644 --- a/tests/travis/ut_install.sh +++ b/tests/travis/ut_install.sh @@ -33,7 +33,7 @@ sudo ./tests/testprepare.sh cd tests && sudo ./ldapprepare.sh && sudo ./admiral.sh && cd .. sudo make compile_adminserver -sudo make -f make/photon/Makefile _build_adminserver _build_db _build_registry -e VERSIONTAG=dev -e CLAIRDBVERSION=dev -e REGISTRYVERSION=${REG_VERSION} +sudo make -f make/photon/Makefile _build_db _build_registry -e VERSIONTAG=dev -e CLAIRDBVERSION=dev -e REGISTRYVERSION=${REG_VERSION} sudo sed -i 's/__reg_version__/${REG_VERSION}-dev/g' ./make/docker-compose.test.yml sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml \ No newline at end of file diff --git a/tests/travis/ut_run.sh b/tests/travis/ut_run.sh index 0ba7a82d43ad..3f21ad2a655c 100755 --- a/tests/travis/ut_run.sh +++ b/tests/travis/ut_run.sh @@ -14,6 +14,6 @@ sleep 10 ./tests/pushimage.sh docker ps -go test -race -i ./src/core ./src/adminserver ./src/jobservice +go test -race -i ./src/core ./src/jobservice sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=/home/travis/gopath/src/github.com/goharbor/harbor/make/migrations/postgresql/" ./tests/coverage4gotest.sh goveralls -coverprofile=profile.cov -service=travis-ci \ No newline at end of file