From 8c2e18f1f3b09b982ea75f95a22b489f4924a9de Mon Sep 17 00:00:00 2001 From: shahram emadi Date: Wed, 26 Aug 2015 15:30:54 -0700 Subject: [PATCH 1/6] Add GCP stuff) --- dev/gcp/README.md | 47 ++++ dev/gcp/build_remote.sh | 13 + dev/gcp/cluster-config-reference.yaml | 35 +++ dev/gcp/cluster-config.yaml | 20 ++ dev/gcp/deploy | 249 ++++++++++++++++++ dev/gcp/functions.sh | 28 ++ dev/gcp/gen-deploy-config.py | 90 +++++++ .../templates/cluster-firewall-template.jinja | 27 ++ .../templates/cluster-instance-template.jinja | 64 +++++ .../templates/cluster-network-template.jinja | 21 ++ .../templates/cluster-setup-template.jinja | 55 ++++ dev/gcp/xe_setup.sh | 5 + 12 files changed, 654 insertions(+) create mode 100644 dev/gcp/README.md create mode 100644 dev/gcp/build_remote.sh create mode 100644 dev/gcp/cluster-config-reference.yaml create mode 100644 dev/gcp/cluster-config.yaml create mode 100755 dev/gcp/deploy create mode 100644 dev/gcp/functions.sh create mode 100644 dev/gcp/gen-deploy-config.py create mode 100644 dev/gcp/templates/cluster-firewall-template.jinja create mode 100644 dev/gcp/templates/cluster-instance-template.jinja create mode 100644 dev/gcp/templates/cluster-network-template.jinja create mode 100644 dev/gcp/templates/cluster-setup-template.jinja create mode 100755 dev/gcp/xe_setup.sh diff --git a/dev/gcp/README.md b/dev/gcp/README.md new file mode 100644 index 000000000000..0ce5500b0c19 --- /dev/null +++ b/dev/gcp/README.md @@ -0,0 +1,47 @@ + +# Levyx deploy test cluster + +### How to use deploy +`deploy` is a shell script that can be invoked to create instances +on Google Cloud Platform (GCP). deploy assumes that: +1. You already have an account on GCP `shared-cluster-0` project +2. You have setup `gcloud` on your computer +3. Current version is limited to Levyx internal use only +4. GCP zone is set to `us-central-1c` + +If you use `deploy` you need not know about the rest of the +scripts in this dir. + +### How to use gen-deploy-config.py +1. Make sure gcloud command line tool is set to the right project + Use ```gcloud config list``` to see the project and user + Use ```gcloud config set project ``` to set the project +2. Make sure the project has the ```levyx-spark-dev-img``` image, it must also + be named exactly as shown. +3. Change directory to the directory with these files. +4. Use: + ``` + python gen-deploy-config.py [arguments] + ``` + All arguments are optional. Type python make-deployment.py -h for help, + or just look at the top of make-deployment.py +5. Use: + ``` + gcloud deployment-manager deployments create levyx-test-deploy --config cluster-config.yaml + ``` +6. Wait for the success message and go to the gcloud console to see the deployment. + The VM will show up in the VM list as well as under "Deploy & Manage > Deployments" + When deleting deployments, delete with + ``` + gcloud deployment-manager deployments delete levyx-test-deploy + ``` + +### Something not working? +Check ```cluster-config-reference``` for an example yaml file that should look +like the yaml file generated by the ```gen-deploy-config.py.``` +```cluster-config-reference``` is NEVER USED BY ANYTHING. It's just for reference. + +Make sure the startup .sh file only has SINGLE quotes ( ' ) NOT double quotes ( " ). +The python script needs to read the .sh file as a string, and thus secretly wraps your +text in double quotes. Having a double quote inside the .sh file will confuse +the script. diff --git a/dev/gcp/build_remote.sh b/dev/gcp/build_remote.sh new file mode 100644 index 000000000000..33395aeb5294 --- /dev/null +++ b/dev/gcp/build_remote.sh @@ -0,0 +1,13 @@ +#!/bin/bash -x + +export LD_LIBRARY_PATH=${HOME}/levyx-spark/xenon/dist/lib +git clone git@github.com:levyx/levyx-spark.git +echo "##################### make -C " +make -C levyx-spark/xenon/dist +pushd levyx-spark/spark +echo "##################### sbt assembly" +sbt assembly +#echo "##################### sbt paraquet" +#sbt "test-only org.apache.spark.sql.parquet.*" +echo "##################### sbt xenon" +sbt "test-only org.apache.spark.sql.xenon.*" diff --git a/dev/gcp/cluster-config-reference.yaml b/dev/gcp/cluster-config-reference.yaml new file mode 100644 index 000000000000..da96e361175b --- /dev/null +++ b/dev/gcp/cluster-config-reference.yaml @@ -0,0 +1,35 @@ +# cluster-config-reference.yaml +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# + +imports: + - path: templates/cluster-instance-template.jinja + - path: templates/cluster-setup-template.jinja + - path: templates/cluster-network-template.jinja + - path: templates/cluster-firewall-template.jinja +resources: + - name: levyxspark-cluster + type: templates/cluster-setup-template.jinja + properties: + # package: levyxspark + projectName: shared-cluster-0 + + instanceImage: global/images/levyx-spark-dev-img + instanceMachineType: n1-standard-8 + instanceNumSSDs: 2 + instanceZone: us-central1-c + instanceBootDiskType: pd-standard-1 + numSlaves: 2 + startupScript: "echo 'hello world' > /home/helloWorld.txt" diff --git a/dev/gcp/cluster-config.yaml b/dev/gcp/cluster-config.yaml new file mode 100644 index 000000000000..7fc181eb4400 --- /dev/null +++ b/dev/gcp/cluster-config.yaml @@ -0,0 +1,20 @@ + +imports: + - path: templates/cluster-instance-template.jinja + - path: templates/cluster-setup-template.jinja + - path: templates/cluster-network-template.jinja + - path: templates/cluster-firewall-template.jinja +resources: + - name: shahram + type: templates/cluster-setup-template.jinja + properties: + # package: levyxspark + projectName: shared-cluster-0 + + instanceImage: global/images/levyx-spark-dev-img + instanceMachineType: n1-standard-16 + instanceNumSSDs: 1 + instanceZone: us-central1-c + instanceBootDiskType: pd-standard-1 + numSlaves: 0 + startupScript: "# nothing" diff --git a/dev/gcp/deploy b/dev/gcp/deploy new file mode 100755 index 000000000000..c2e136380c72 --- /dev/null +++ b/dev/gcp/deploy @@ -0,0 +1,249 @@ +#!/bin/bash +# +# deploy.sh +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# +# Deploy instance on Google cloud and clone the levyx-spark repo +# +# Assumptions: +# 1. The instance is created on shared-cluster-0 of Levyx +# 2. The instance is in us-central1-c zone + +# - Check gcloud version with deployment manager +# - Set git project +# - Create yaml +# - Create instance +# - Display ssh credentials +# +source functions.sh + +usage() { +cat << EOF + + deploy [--help] + deploy create [--project|--instance|--name|--quiet|--clone] + deploy remove name + deploy list + + deploy create + --project GCP project name (Default: shared-cluster-0) + --instance GCP instance type (Default: n1-standard-16 + 1 local SSD) + --name GCP instance host prefix (Default: username) + --quiet Disable verbose output when creating GCP instance + --clone Clone levyx-spark from GitHub (requires ssh keys) + + deploy remove [--project] + --project GCP project name (Default: shared-cluster-0) + name GCP instance name, typically username-spark-test + + deploy list [--project] + --project GCP project name (Default: shared-cluster-0) + +EOF + exit 1 +} + +DEFAULT_PROJECT=shared-cluster-0 +DEFAULT_INSTANCE=n1-standard-16 +DEFAULT_SSD=1 +DEFAULT_PREFIX=`basename $HOME` +DEFAULT_SLAVES=0 + +PGNAME="${YELLOW}[deploy]${NORM}" +# Assumptions: +# 1. For this version, we have hardcoded SSDs == 1. This is +# so that we do not have to deal with creating RAID on +# the cloud machine and simply the instance creation. +# 2. Number of slaves == 0 => Single node instance +PROJECT="" +INSTANCE="" +NUM_SSD=$DEFAULT_SSD +PREFIX="" +CLONE=0 +SLAVES=$DEFAULT_SLAVES + +GCP_YAML_SCRIPT="python ./gen-deploy-config.py" +YAML_CFG_FILE=cluster-config.yaml +GIT_CLONE_CMD="git clone https://github.com/levyx/levyx-spark" + +DO_CREATE="" +DO_REMOVE="" +DO_LIST="" + +sanity() { + which gcloud >/dev/null 2>&1 \ + || err_exit "gcloud not found" + + gcloud deployment-manager -h >/dev/null 2>&1 || \ + err_exit "gcloud deployment-manager not found, upgrade gcloud" + + [[ -z $PROJECT ]] && PROJECT=$DEFAULT_PROJECT + [[ -z $INSTANCE ]] && INSTANCE=$DEFAULT_INSTANCE + [[ -z $PREFIX ]] && PREFIX=$DEFAULT_PREFIX +} + +print_config() { + echo "Cluster configuration:" + echo " Project: $BOLD $PROJECT $NORM" + echo " Instance: $BOLD $INSTANCE $NORM" + echo " SSDs: $BOLD $NUM_SSD $NORM" + echo "" + echo "Google cloud configuration:" + gcloud config list + echo "" +} + +msg() { + echo "${PGNAME} $*" + echo "" +} + +banner() { + echo -n "${YELLOW}" +cat << EOF + o + | Levyx Spark Cluster Deployment + o--o +EOF + echo "${NORM}" +} + +create() +{ + sanity + + # Unfortunately gcloud config does not return !0 for an + # invalid project => cannot check for $? correctness + msg "Setting gcloud project to ${BOLD}${PROJECT} $NORM" + gcloud config set project $PROJECT + gcloud config set compute/zone us-central1-c + + print_config + + $GCP_YAML_SCRIPT -t $INSTANCE -d $NUM_SSD -s $SLAVES -n $PREFIX -r xe_setup.sh >/dev/null + if [ $? -ne 0 ]; then + err_exit "$GCP_YAML_SCRIPT failed" + fi + msg "Generated gcloud cluster config .yaml file" + ls -ltr *.yaml | tail -1 + + msg "Creating instance" + gcloud deployment-manager deployments create \ + ${PREFIX}-spark-test --config $YAML_CFG_FILE + if [ $? -ne 0 ]; then + echo "" + err_exit "Please check gcloud error message..." + fi + + msg "Listing instance properties" + gcloud compute instances list | grep ${PREFIX} + + if [ $CLONE -eq 1 ]; then + msg "Clone git repo ..." + gcloud compute ssh ${PREFIX}-master '$GIT_CLONE_CMD' + if [ $? -ne 0 ]; then + err_exit "$GIT_CLONE_CMD failed" + fi + fi + + msg "Useful commands:" + echo " ${BOLD}[ssh]${NORM} gcloud compute ssh ${PREFIX}-master" + echo " ${BOLD}[copy]${NORM} gcloud compute copy-files" + echo " ${BOLD}[rexec]${NORM} gcloud compute ssh ${PREFIX}-master 'remote command'" + echo "" +} + +remove() { + sanity + + # Unfortunately gcloud config does not return !0 for an + # invalid project => cannot check for $? correctness + msg "Setting gcloud project to ${BOLD}${PROJECT} $NORM" + gcloud config set project $PROJECT + gcloud config set compute/zone us-central1-c + + gcloud deployment-manager deployments delete -q $INSTANCE +} + +list() { + sanity + + # Unfortunately gcloud config does not return !0 for an + # invalid project => cannot check for $? correctness + msg "Setting gcloud project to ${BOLD}${PROJECT} $NORM" + gcloud config set project $PROJECT + gcloud config set compute/zone us-central1-c + + gcloud deployment-manager deployments list +} + +if [ $# -eq 0 ]; then + usage +fi + +case $1 in + create) DO_CREATE="true" ;; + remove) DO_REMOVE="true" ;; + list) DO_LIST="true" ;; + *) usage ;; +esac +shift + +if [ -n "$DO_CREATE" ]; then + if [ $# -eq 0 ]; then + echo "$PGNAME No args specified. Using defaults ..." + echo "" + fi + while [ "$#" -gt 0 ]; do + case $1 in + --project) PROJECT="$2"; shift ;; + --instance) INSTANCE="$2"; shift ;; + --name) PREFIX="$2"; shift ;; + --clone) CLONE=1; ;; + -h|--help) usage ;; + *) usage ;; + esac + shift + done + banner + create +fi + +if [ -n "$DO_REMOVE" ]; then + if [ $# -eq 0 ]; then + echo "$PGNAME Error: No instance specified!" + echo "" + usage + fi + while [ "$#" -gt 0 ]; do + case $1 in + --project) PROJECT="$2"; INSTANCE="$3"; break ;; + *) INSTANCE="$1"; break ;; + esac + done + banner + remove +fi + +if [ -n "$DO_LIST" ]; then + while [ "$#" -gt 0 ]; do + case $1 in + --project) PROJECT="$2"; break ;; + esac + shift + done + banner + list +fi diff --git a/dev/gcp/functions.sh b/dev/gcp/functions.sh new file mode 100644 index 000000000000..0108f8f53ed4 --- /dev/null +++ b/dev/gcp/functions.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# functions.sh +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# +# Utility functions +# + +err_exit() { + echo "Error: $*" + exit -1 +} + +BOLD=$(tput bold) +NORM=$(tput sgr0) +YELLOW=$(tput setaf 3) diff --git a/dev/gcp/gen-deploy-config.py b/dev/gcp/gen-deploy-config.py new file mode 100644 index 000000000000..1c96360f0de3 --- /dev/null +++ b/dev/gcp/gen-deploy-config.py @@ -0,0 +1,90 @@ +# gen-deploy-config.py +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# +# Generates YAML for gcloud deployment-manager +# Requires Python 2.7 or above +# + +import argparse +import sys + +parser = argparse.ArgumentParser( + description="Script to generate YAML config", + add_help=False + ) +parser.add_argument("-h", "--help", action="help", help="Print this help message and exit." ) +parser.add_argument("-n", "--name", help="Prefix for the names of the instances.", metavar="", default="levyxspark-cluster", required=False) +parser.add_argument("-t", "--type", help="Type of machine.", metavar="", default="n1-standard-16", required=False) +parser.add_argument("-d", "--ssds", help="Number of SSDs.", metavar="", default="1", required=False) +parser.add_argument("-s", "--slaves", help="Number of slaves.", metavar="", default="0", required=False) +parser.add_argument("-r", "--startup", help="Startup script file", metavar="", default="", required=False) +args = parser.parse_args() + +reqPythonVersion = (2,7) +curPythonVersion = sys.version_info + +if curPythonVersion < reqPythonVersion: + print "\nERROR: Python version must be 2.7 or above, you are running {0}.\n".format(curPythonVersion) + sys.exit() + +# ============================================================================ +# Load any arguments into variables +# ============================================================================ + +instanceName = args.name +machineType = args.type +numSSDs = args.ssds +numSlaves = args.slaves + +startupScript = "# nothing" +if ( len( args.startup ) > 0 ): + with open ( args.startup, 'r') as startupFile: + startupScript = startupFile.read().strip().replace('\n', " && ") +startupScript = '"{}"'.format( startupScript ) +# ============================================================================ +# The actual YAML +# ============================================================================ +data = """ +imports: + - path: templates/cluster-instance-template.jinja + - path: templates/cluster-setup-template.jinja + - path: templates/cluster-network-template.jinja + - path: templates/cluster-firewall-template.jinja +resources: + - name: %(instanceName)s + type: templates/cluster-setup-template.jinja + properties: + # package: levyxspark + projectName: shared-cluster-0 + + instanceImage: global/images/levyx-spark-dev-img + instanceMachineType: %(machineType)s + instanceNumSSDs: %(numSSDs)s + instanceZone: us-central1-c + instanceBootDiskType: pd-standard-1 + numSlaves: %(numSlaves)s + startupScript: %(startupScript)s +"""% { "instanceName": instanceName, + "machineType": machineType, + "numSSDs": numSSDs, + "numSlaves": numSlaves, + "startupScript": startupScript } + +# ============================================================================ +# Write to file +# ============================================================================ +with open("cluster-config.yaml", 'w') as outfile: + outfile.write( data ) +print "created cluster-config.yaml" diff --git a/dev/gcp/templates/cluster-firewall-template.jinja b/dev/gcp/templates/cluster-firewall-template.jinja new file mode 100644 index 000000000000..e17eebdc5c35 --- /dev/null +++ b/dev/gcp/templates/cluster-firewall-template.jinja @@ -0,0 +1,27 @@ +# cluster-firewall-template.jinja +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# + +{% set network = properties["instanceNetwork"] %} + +resources: +- name: {{ env["name"] }} + type: compute.v1.firewall + properties: + network: $(ref.{{ network }}.selfLink) + sourceRanges: ["0.0.0.0/0"] + allowed: + - IPProtocol: TCP + ports: ["8080", "4040", "22", "80", "433"] diff --git a/dev/gcp/templates/cluster-instance-template.jinja b/dev/gcp/templates/cluster-instance-template.jinja new file mode 100644 index 000000000000..1742c9055b45 --- /dev/null +++ b/dev/gcp/templates/cluster-instance-template.jinja @@ -0,0 +1,64 @@ +# cluster-instance.jinja +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# + +{% set image = properties["instanceImage"] %} +{% set network = properties["instanceNetwork"] %} +{% set zone = properties["instanceZone"] %} +{% set bootDiskType = properties["instanceBootDiskType"] %} +{% set machineType = properties["instanceMachineType"] %} +{% set numSSDs = properties["instanceNumSSDs"] %} +{% set projectName = properties["projectName"] %} +{% set startupScript = properties["startupScript"] %} + +resources: +- type: compute.v1.instance + name: {{ env["name"] }} + properties: + network: $(ref.{{ network }}.selfLink) + zone: {{ zone }} + machineType: zones/{{ zone }}/machineTypes/{{ machineType }} + canIPForward: true + + networkInterfaces: + - network: $(ref.{{ network }}.selfLink) + accessConfigs: + - name: External NAT + type: ONE_TO_ONE_NAT + + disks: + - deviceName: boot + type: PERSISTENT + diskType: {{ bootDiskType }} + diskSizeGb: 10 + boot: true + autoDelete: true + initializeParams: + sourceImage: "{{ image }}" + + + {% for i in range(0,numSSDs) %} + - type: SCRATCH + interface: SCSI + mode: READ_WRITE + autoDelete: true + initializeParams: + diskType: zones/us-central1-c/diskTypes/local-ssd + {% endfor %} + + metadata: + items: + - key: startup-script + value: {{startupScript}} diff --git a/dev/gcp/templates/cluster-network-template.jinja b/dev/gcp/templates/cluster-network-template.jinja new file mode 100644 index 000000000000..73dacfcadebd --- /dev/null +++ b/dev/gcp/templates/cluster-network-template.jinja @@ -0,0 +1,21 @@ +# cluster-network-template.jinja +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# + +resources: +- name: {{ env["name"] }} + type: compute.v1.network + properties: + IPv4Range: 10.0.0.1/16 diff --git a/dev/gcp/templates/cluster-setup-template.jinja b/dev/gcp/templates/cluster-setup-template.jinja new file mode 100644 index 000000000000..d5baff0cf0d8 --- /dev/null +++ b/dev/gcp/templates/cluster-setup-template.jinja @@ -0,0 +1,55 @@ +# cluster-setup-template.jinja +# +# Copyright (C) 2013-2015, Levyx, Inc. +# +# NOTICE: All information contained herein is, and remains the property of +# Levyx, Inc. The intellectual and technical concepts contained herein are +# proprietary to Levyx, Inc. and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material is +# strictly forbidden unless prior written permission is obtained from Levyx, +# Inc. Access to the source code contained herein is hereby forbidden to +# anyone except current Levyx, Inc. employees, managers or contractors who +# have executed Confidentiality and Non-disclosure agreements explicitly +# covering such access. +# + +{% set image = properties["instanceImage"] %} +{% set zone = properties["instanceZone"] %} +{% set bootDiskType = properties["instanceBootDiskType"] %} +{% set machineType = properties["instanceMachineType"] %} +{% set numSSDs = properties["instanceNumSSDs"] %} +{% set projectName = properties["projectName"] %} +{% set numSlaves = properties["numSlaves"] %} +{% set startupScript = properties["startupScript"] %} + +resources: +- name: {{ env["name"] }}-master + type: templates/cluster-instance-template.jinja + properties: + instanceImage: {{image}} + instanceNetwork: {{ env["name"] }}-network + instanceMachineType: {{machineType}} + instanceNumSSDs: {{numSSDs}} + instanceZone: {{zone}} + instanceBootDiskType: {{bootDiskType}} + startupScript: {{startupScript}} + +{% for i in range( numSlaves ) %} +- name: {{ env["name"] }}-slave-{{i}} + type: templates/cluster-instance-template.jinja + properties: + instanceImage: {{image}} + instanceNetwork: {{ env["name"] }}-network + instanceMachineType: {{machineType}} + instanceNumSSDs: {{numSSDs}} + instanceZone: {{zone}} + instanceBootDiskType: {{bootDiskType}} + startupScript: {{startupScript}} +{% endfor %} +- name: {{ env["name"] }}-network + type: templates/cluster-network-template.jinja +- name: {{ env["name"] }}-firewall + type: templates/cluster-firewall-template.jinja + properties: + instanceNetwork: {{ env["name"] }}-network diff --git a/dev/gcp/xe_setup.sh b/dev/gcp/xe_setup.sh new file mode 100755 index 000000000000..b80d73cfb5ce --- /dev/null +++ b/dev/gcp/xe_setup.sh @@ -0,0 +1,5 @@ +touch /tmp/xe_test +mkfs.xfs /dev/sdb +mkdir /mnt/xenon/ +mount /dev/sdb /mnt/xenon/ +chmod -R 0777 /mnt/xenon From 31095671fde7b70de9f498ff048790872f1157c7 Mon Sep 17 00:00:00 2001 From: semad Date: Wed, 26 Aug 2015 16:05:18 -0700 Subject: [PATCH 2/6] Update build_remote.sh --- dev/gcp/build_remote.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/gcp/build_remote.sh b/dev/gcp/build_remote.sh index 33395aeb5294..9b024a8c4331 100644 --- a/dev/gcp/build_remote.sh +++ b/dev/gcp/build_remote.sh @@ -1,13 +1,13 @@ #!/bin/bash -x export LD_LIBRARY_PATH=${HOME}/levyx-spark/xenon/dist/lib -git clone git@github.com:levyx/levyx-spark.git -echo "##################### make -C " -make -C levyx-spark/xenon/dist -pushd levyx-spark/spark +git clone https://github.com/semad/spark.git +#echo "##################### make -C " +#make -C levyx-spark/xenon/dist +pushd ./spark echo "##################### sbt assembly" sbt assembly #echo "##################### sbt paraquet" #sbt "test-only org.apache.spark.sql.parquet.*" -echo "##################### sbt xenon" -sbt "test-only org.apache.spark.sql.xenon.*" +#echo "##################### sbt xenon" +#sbt "test-only org.apache.spark.sql.xenon.*" From a091f158be9ae3189a37670866434a575bd0d968 Mon Sep 17 00:00:00 2001 From: semad Date: Wed, 26 Aug 2015 16:22:05 -0700 Subject: [PATCH 3/6] Update build_remote.sh --- dev/gcp/build_remote.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/gcp/build_remote.sh b/dev/gcp/build_remote.sh index 9b024a8c4331..84f47b8de322 100644 --- a/dev/gcp/build_remote.sh +++ b/dev/gcp/build_remote.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash export LD_LIBRARY_PATH=${HOME}/levyx-spark/xenon/dist/lib git clone https://github.com/semad/spark.git From 03e7d292a795b1d16f5c426b989d18cd9f86cf28 Mon Sep 17 00:00:00 2001 From: semad Date: Wed, 26 Aug 2015 16:29:21 -0700 Subject: [PATCH 4/6] Update README.md Test commits --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 380422ca00db..0fa47dea8d0e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Apache Spark + + Spark is a fast and general cluster computing system for Big Data. It provides high-level APIs in Scala, Java, and Python, and an optimized engine that supports general computation graphs for data analysis. It also supports a From 9e582a065207db0b13aa146fef987bf1e52754fa Mon Sep 17 00:00:00 2001 From: semad Date: Wed, 26 Aug 2015 16:54:27 -0700 Subject: [PATCH 5/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0fa47dea8d0e..ff4f5cf5a13f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Apache Spark - +# Added for test Spark is a fast and general cluster computing system for Big Data. It provides high-level APIs in Scala, Java, and Python, and an optimized engine that From e39a30b1166d16fabff797781fcfce4eb732ae93 Mon Sep 17 00:00:00 2001 From: semad Date: Thu, 27 Aug 2015 09:27:59 -0700 Subject: [PATCH 6/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff4f5cf5a13f..aa1b8b3cefd1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Apache Spark -# Added for test +# Added PR Spark is a fast and general cluster computing system for Big Data. It provides high-level APIs in Scala, Java, and Python, and an optimized engine that