[DO NOT merge] Thin opensds

contrib/backup/multicloud/driver.go

@@ -20,10 +20,11 @@ import (
 	"io/ioutil"
 	"os"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/golang/glog"
 	"github.com/opensds/opensds/contrib/backup"
 	"github.com/opensds/opensds/pkg/utils"
-	"gopkg.in/yaml.v2"
 )
 
 const (

contrib/drivers/ceph/ceph_metrics_test.go

@@ -60,19 +60,19 @@ var expected_data map[string]CephMetricStats = map[string]CephMetricStats{
 	"osd_total_used_bytes":     {"total_used", "114624000", "bytes", nil, "", health_label, "osd"},
 	"osd_total_avail_bytes":    {"total_avail", "15602732000", "bytes", nil, "", health_label, "osd"},
 	"osd_average_utilization":  {"average_utilization", "0.729283", "", nil, "", health_label, "osd"},
-	"osd":                      {"osd", "0", "", nil, "", osd_label, ""},
-	"osd_up":                   {"osd_up", "1", "", nil, "", osd_label, ""},
-	"osd_in":                   {"osd_in", "1", "", nil, "", osd_label, ""},
-	"client_io_write_ops":      {"io_write", "0", "ops", nil, "", health_label, "client"},
-	"client_io_read_bytes":     {"io_read", "0", "bytes", nil, "", health_label, "client"},
-	"client_io_read_ops":       {"io_read", "0", "ops", nil, "", health_label, "client"},
-	"client_io_write_bytes":    {"io_write", "0", "bytes", nil, "", health_label, "client"},
-	"cache_flush_io_bytes":     {"cache_flush_io", "0", "bytes", nil, "", health_label, ""},
-	"cache_evict_io_bytes":     {"cache_evict_io", "0", "bytes", nil, "", health_label, ""},
-	"cache_promote_io_ops":     {"cache_promote_io", "0", "ops", nil, "", health_label, ""},
-	"degraded_objects":         {"degraded_objects", "0", "", nil, "", health_label, ""},
-	"misplaced_objects":        {"misplaced_objects", "0", "", nil, "", health_label, ""},
-	"osds":                     {"osds", "1", "", nil, "", health_label, ""}, "osds_up": {"osds_up", "1", "", nil, "", health_label, ""}, "osds_in": {"osds_in", "1", "", nil, "", health_label, ""},
+	"osd":                   {"osd", "0", "", nil, "", osd_label, ""},
+	"osd_up":                {"osd_up", "1", "", nil, "", osd_label, ""},
+	"osd_in":                {"osd_in", "1", "", nil, "", osd_label, ""},
+	"client_io_write_ops":   {"io_write", "0", "ops", nil, "", health_label, "client"},
+	"client_io_read_bytes":  {"io_read", "0", "bytes", nil, "", health_label, "client"},
+	"client_io_read_ops":    {"io_read", "0", "ops", nil, "", health_label, "client"},
+	"client_io_write_bytes": {"io_write", "0", "bytes", nil, "", health_label, "client"},
+	"cache_flush_io_bytes":  {"cache_flush_io", "0", "bytes", nil, "", health_label, ""},
+	"cache_evict_io_bytes":  {"cache_evict_io", "0", "bytes", nil, "", health_label, ""},
+	"cache_promote_io_ops":  {"cache_promote_io", "0", "ops", nil, "", health_label, ""},
+	"degraded_objects":      {"degraded_objects", "0", "", nil, "", health_label, ""},
+	"misplaced_objects":     {"misplaced_objects", "0", "", nil, "", health_label, ""},
+	"osds":                  {"osds", "1", "", nil, "", health_label, ""}, "osds_up": {"osds_up", "1", "", nil, "", health_label, ""}, "osds_in": {"osds_in", "1", "", nil, "", health_label, ""},
 	"pgs_remapped": {"pgs_remapped", "0", "", nil, "", health_label, ""}, "total_pgs": {"total_pgs", "102", "", nil, "", health_label, ""}, "volume_name": {"name", "opensds-4c5cb264-50d1-4bfd-a663-dface9b669c9", "", nil, "", volume_label, "volume"},
 	"volume_size_bytes":        {"size", "1073741824", "bytes", nil, "", volume_label, "volume"},
 	"volume_objects":           {"objects", "1024", "", nil, "", volume_label, "volume"},

contrib/drivers/huawei/oceanstor/metrics.go

@@ -18,12 +18,13 @@ import (
 	"strconv"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	log "github.com/golang/glog"
 	. "github.com/opensds/opensds/contrib/drivers/utils/config"
 	"github.com/opensds/opensds/pkg/model"
 	"github.com/opensds/opensds/pkg/utils/config"
 	uuid "github.com/satori/go.uuid"
-	"gopkg.in/yaml.v2"
 )
 
 /*

contrib/drivers/lvm/lvm_metrics.go

@@ -19,9 +19,10 @@ import (
 	"strings"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	log "github.com/golang/glog"
 	"github.com/opensds/opensds/pkg/model"
-	"gopkg.in/yaml.v2"
 )
 
 // Supported metrics

contrib/drivers/utils/config/config.go

@@ -22,9 +22,10 @@ package config
 import (
 	"io/ioutil"
 
+	"gopkg.in/yaml.v2"
+
 	log "github.com/golang/glog"
 	"github.com/opensds/opensds/pkg/model"
-	"gopkg.in/yaml.v2"
 )
 
 type PoolProperties struct {

docs/Thin-OpenSDS Installation Guide.md

@@ -0,0 +1,117 @@
+# Thin OpenSDS Installation Guide
+## Prepare
+Before you start, please make sure you have all the following:
+- Ubuntu environment (suggest v16.04+).
+- More than 30GB remaining disk.
+- Make sure have access to the Internet.
+- Some tools (`git`, `make` and `gcc`) prepared.
+
+#### Bootstrap
+First you need to download [bootstrap](https://github.com/opensds/opensds/blob/thin-opensds/install/devsds/bootstrap.sh) script and run it locally with root access.
+```shell
+curl -sSL https://raw.githubusercontent.com/opensds/opensds/thin-opensds/install/devsds/bootstrap.sh | sudo bash
+```
+If there is no error reported, you have all dependency packages installed.
+
+#### Authentication configuration
+Because the default authentication strategy is `noauth`, so if you want to enable multi-tenants feature or want to use Dashboard, please set the field `OPENSDS_AUTH_STRATEGY=keystone` in local.conf file:
+```shell
+cd $GOPATH/src/github.com/opensds/opensds
+vi install/devsds/local.conf
+```
+
+#### Run osdsapiserver and osdsdock services of OpenSDS using following
+```
+cd $GOPATH/src/github.com/opensds/opensds && install/devsds/install.sh
+```
+If everything goes well, you will get some connection messages at the console output:
+```shell
+Execute commands below to set up ENVs which are needed by OpenSDS CLI:
+------------------------------------------------------------------
+export OPENSDS_AUTH_STRATEGY=keystone
+export OPENSDS_ENDPOINT=http://localhost:50040
+export OS_AUTH_URL=http://<host_ip>/identity
+export OS_USERNAME=admin
+export OS_PASSWORD=opensds@123
+export OS_TENANT_NAME=admin
+export OS_PROJECT_NAME=admin
+export OS_USER_DOMAIN_ID=default
+------------------------------------------------------------------
+```
+
+## Testing
+
+### Testing OpenSDS(Hotpot) using CLI
+#### Config osdsctl tool
+```shell
+sudo cp $GOPATH/src/github.com/opensds/opensds/build/out/bin/osdsctl /usr/local/bin
+```
+
+#### Set some environment variables
+```shell
+export OPENSDS_ENDPOINT=http://127.0.0.1:50040
+export OPENSDS_AUTH_STRATEGY=noauth # Set the value to keystone for multi-tenants.
+```
+
+If you choose keystone for authentication strategy, you need to execute different commands for logging in as different roles:
+* For admin role
+```shell
+export OPENSDS_AUTH_STRATEGY=keystone
+export OPENSDS_ENDPOINT=http://localhost:50040
+export OS_AUTH_URL=http://$HOST_IP/identity
+export OS_USERNAME=admin
+export OS_PASSWORD=opensds@123
+export OS_TENANT_NAME=admin
+export OS_PROJECT_NAME=admin
+export OS_USER_DOMAIN_ID=default
+```
+
+#### Create a volume using create volume API request
+##### Endpoint
+
+```$xslt
+http://localhost:50040/v1beta/{{ tenant ID }}/block/volumes
+```
+
+##### Request body (update poolName and poolID)
+```
+{
+  "name": "test-1",
+  "description": "volume create test through api",
+  "size": 1,
+  "availabilityZone": "default",
+  "profileId": "",
+  "poolId":"{{ pool ID }}",
+  "metadata": {
+  	"poolName": "opensds-volumes"
+  },
+  "snapshotFromCloud": false
+}
+```
+
+#### List all volumes
+
+```
+osdsctl volume list
+```
+
+#### Delete the volume
+```
+osdsctl volume delete <your_volume_id>
+```
+
+
+You can install [CSI Plugin using Helm](https://github.com/opensds/opensds-installer/blob/master/charts/OpenSDS%20Installation%20using%20Helm.md)
+## Uninstall the local cluster
+### OpenSDS(Hotpot)
+It's also nice to uninstall the cluster with one command:
+```
+cd $GOPATH/src/github.com/opensds/opensds && install/devsds/uninstall.sh
+```
+
+If you want to destroy the cluster, please run the command below instead:
+```
+cd $GOPATH/src/github.com/opensds/opensds && install/devsds/uninstall.sh -purge
+```
+
+

examples/opensds.conf

@@ -23,6 +23,8 @@ beego_https_cert_file =
 beego_https_key_file =
 # Encryption and decryption tool. Default value is aes.
 password_decrypt_tool = aes
+# Choose "thin" for Thin OpenSDS and "full" for Full OpenSDS services
+install_type = full
 
 [osdslet]
 api_endpoint = localhost:50049

install/devsds/bootstrap.sh

@@ -78,7 +78,7 @@ mkdir -p ${OPENSDS_ROOT}
 cd ${OPENSDS_ROOT}
 if [ ! -d ${OPENSDS_DIR} ]; then
     log "Downloading the OpenSDS source code..."
-    git clone https://github.com/opensds/opensds.git -b master
+    git clone https://github.com/opensds/opensds.git -b thin-opensds
 fi
 
 # make sure 'make' has been installed.

install/devsds/lib/opensds.sh

@@ -32,9 +32,10 @@ auth_strategy = $OPENSDS_AUTH_STRATEGY
 https_enabled = False
 beego_https_cert_file =
 beego_https_key_file =
+install_type = thin
 
-[osdslet]
-api_endpoint = $HOST_IP:50049
+#[osdslet]
+#api_endpoint = $HOST_IP:50049
 
 [osdsdock]
 api_endpoint = $HOST_IP:50050
@@ -54,7 +55,7 @@ osds::opensds::install(){
 (
     cd ${OPENSDS_DIR}
     sudo build/out/bin/osdsapiserver --daemon
-    sudo build/out/bin/osdslet --daemon
+   #sudo build/out/bin/osdslet --daemon
     sudo build/out/bin/osdsdock --daemon
 
     osds::echo_summary "Waiting for osdsapiserver to come up."
@@ -93,7 +94,7 @@ osds::opensds::install(){
 }
 
 osds::opensds::cleanup() {
-    sudo killall -9 osdsapiserver osdslet osdsdock &>/dev/null
+    sudo killall -9 osdsapiserver osdsdock &>/dev/null
 }
 
 osds::opensds::uninstall(){

install/tools/pwdEncrypter/passwordencrypter.go

@@ -18,11 +18,12 @@ import (
 	"fmt"
 	"os"
 
+	"gopkg.in/yaml.v2"
+
 	"io/ioutil"
 
 	"github.com/opensds/opensds/pkg/utils/pwd"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 )
 
 const (