Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
Merge pull request #2 from apache/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@bodnarmykola
bodnarmykola committed Jul 19, 2019
2 parents 21a9bb0 + 84d0115 commit 75c4b3b
Show file tree
Hide file tree
Showing 382 changed files with 11,584 additions and 7,609 deletions.
583 changes: 334 additions & 249 deletions README.md
View file

Large diffs are not rendered by default.

17 changes: 9 additions & 8 deletions infrastructure-provisioning/scripts/deploy_dlab.py
View file
Expand Up @@ -77,6 +77,7 @@
parser.add_argument('--conf_key_name', type=str, default='', help='Admin key name (WITHOUT ".pem")')
parser.add_argument('--workspace_path', type=str, default='', help='Admin key name (WITHOUT ".pem")')
parser.add_argument('--conf_tag_resource_id', type=str, default='dlab', help='The name of user tag')
parser.add_argument('--conf_billing_tag', type=str, default='dlab', help='Billing tag')
parser.add_argument('--aws_ssn_instance_size', type=str, default='t2.large', help='The SSN instance shape')
parser.add_argument('--azure_ssn_instance_size', type=str, default='Standard_DS2_v2', help='The SSN instance shape')
parser.add_argument('--gcp_ssn_instance_size', type=str, default='n1-standard-2', help='The SSN instance shape')
Expand All @@ -103,18 +104,18 @@
parser.add_argument('--azure_source_resource_group_name', type=str, default='', help='Azure source resource group')
parser.add_argument('--gcp_project_id', type=str, default='', help='The project ID in Google Cloud Platform')
parser.add_argument('--gcp_service_account_path', type=str, default='', help='The project ID in Google Cloud Platform')
parser.add_argument('--dlab_id', type=str, default="'user:user:tag'", help='Column name in report file that contains '
parser.add_argument('--dlab_id', type=str, default="'resource_tags_user_user_tag'", help='Column name in report file that contains '
'dlab id tag')
parser.add_argument('--usage_date', type=str, default='UsageStartDate', help='Column name in report file that contains '
parser.add_argument('--usage_date', type=str, default='line_item_usage_start_date', help='Column name in report file that contains '
'usage date tag')
parser.add_argument('--product', type=str, default='ProductName', help='Column name in report file that contains '
parser.add_argument('--product', type=str, default='product_product_name', help='Column name in report file that contains '
'product name tag')
parser.add_argument('--usage_type', type=str, default='UsageType', help='Column name in report file that contains '
parser.add_argument('--usage_type', type=str, default='line_item_usage_type', help='Column name in report file that contains '
'usage type tag')
parser.add_argument('--usage', type=str, default='UsageQuantity', help='Column name in report file that contains '
parser.add_argument('--usage', type=str, default='line_item_usage_amount', help='Column name in report file that contains '
'usage tag')
parser.add_argument('--cost', type=str, default='BlendedCost', help='Column name in report file that contains cost tag')
parser.add_argument('--resource_id', type=str, default='ResourceId', help='Column name in report file that contains '
parser.add_argument('--cost', type=str, default='line_item_blended_cost', help='Column name in report file that contains cost tag')
parser.add_argument('--resource_id', type=str, default='line_item_resource_id', help='Column name in report file that contains '
'dlab resource id tag')
parser.add_argument('--ldap_hostname', type=str, default='localhost', help='Ldap instance hostname')
parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
Expand All @@ -123,7 +124,7 @@
parser.add_argument('--ldap_service_username', type=str, default='cn=service-user', help='Ldap service user name')
parser.add_argument('--ldap_service_password', type=str, default='service-user-password',
help='Ldap password for admin user')
parser.add_argument('--tags', type=str, default='Operation,ItemDescription', help='Column name in report file that '
parser.add_argument('--tags', type=str, default='line_item_operation,line_item_line_item_description', help='Column name in report file that '
'contains tags')
parser.add_argument('--action', required=True, type=str, default='', choices=['build', 'deploy', 'create', 'terminate'],
help='Available options: build, deploy, create, terminate')
Expand Down
5 changes: 4 additions & 1 deletion infrastructure-provisioning/src/base/scripts/install_user_key.py
View file
Expand Up @@ -47,7 +47,10 @@ def copy_key(config):
config.get('user_keydir'),
config.get('user_keyname'))
print(user_key)
key = open('{0}'.format(user_key)).read()
try:
key = config.get('user_key')
except KeyError:
key = open('{0}'.format(user_key)).read()
sudo('echo "{0}" >> /home/{1}/.ssh/authorized_keys'.format(key, args.user))
except:
print('No user key')
Expand Down
145 changes: 72 additions & 73 deletions infrastructure-provisioning/src/edge/fabfile.py
View file
Expand Up @@ -45,47 +45,47 @@ def status():
sys.exit(1)


def run():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)

try:
local("~/scripts/{}.py".format('edge_prepare'))
except Exception as err:
traceback.print_exc()
append_result("Failed preparing Edge node.", str(err))
sys.exit(1)

try:
local("~/scripts/{}.py".format('edge_configure'))
except Exception as err:
traceback.print_exc()
append_result("Failed configuring Edge node.", str(err))
sys.exit(1)
#def run():
# local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
# os.environ['request_id'])
# local_log_filepath = "/logs/edge/" + local_log_filename
# logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
# level=logging.DEBUG,
# filename=local_log_filepath)
#
# try:
# local("~/scripts/{}.py".format('edge_prepare'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed preparing Edge node.", str(err))
# sys.exit(1)
#
# try:
# local("~/scripts/{}.py".format('edge_configure'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed configuring Edge node.", str(err))
# sys.exit(1)


# Main function for terminating EDGE node and exploratory environment if exists
def terminate():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)
try:
local("~/scripts/{}.py".format('edge_terminate'))
except Exception as err:
traceback.print_exc()
append_result("Failed terminating Edge node.", str(err))
sys.exit(1)
#def terminate():
# local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
# local_log_filepath = "/logs/edge/" + local_log_filename
# logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
# level=logging.DEBUG,
# filename=local_log_filepath)
# try:
# local("~/scripts/{}.py".format('edge_terminate'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed terminating Edge node.", str(err))
# sys.exit(1)


# Main function for stopping EDGE node
def stop():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'], os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
Expand All @@ -98,9 +98,9 @@ def stop():
sys.exit(1)


# Main function for stopping EDGE node
# Main function for starting stoped EDGE node
def start():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'], os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
Expand All @@ -113,40 +113,39 @@ def start():
sys.exit(1)


def recreate():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)

try:
local("~/scripts/{}.py".format('edge_prepare'))
except Exception as err:
traceback.print_exc()
append_result("Failed preparing Edge node.", str(err))
sys.exit(1)

try:
local("~/scripts/{}.py".format('edge_configure'))
except Exception as err:
traceback.print_exc()
append_result("Failed configuring Edge node.", str(err))
sys.exit(1)


def reupload_key():
local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
os.environ['request_id'])
local_log_filepath = "/logs/edge/" + local_log_filename
logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
level=logging.DEBUG,
filename=local_log_filepath)

try:
local("~/scripts/{}.py".format('reupload_ssh_key'))
except Exception as err:
traceback.print_exc()
append_result("Failed to reupload key on Edge node.", str(err))
sys.exit(1)
#def recreate():
# local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'],
# os.environ['request_id'])
# local_log_filepath = "/logs/edge/" + local_log_filename
# logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
# level=logging.DEBUG,
# filename=local_log_filepath)
#
# try:
# local("~/scripts/{}.py".format('edge_prepare'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed preparing Edge node.", str(err))
# sys.exit(1)
#
# try:
# local("~/scripts/{}.py".format('edge_configure'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed configuring Edge node.", str(err))
# sys.exit(1)

#def reupload_key():
# local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
# os.environ['request_id'])
# local_log_filepath = "/logs/edge/" + local_log_filename
# logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
# level=logging.DEBUG,
# filename=local_log_filepath)
#
# try:
# local("~/scripts/{}.py".format('reupload_ssh_key'))
# except Exception as err:
# traceback.print_exc()
# append_result("Failed to reupload key on Edge node.", str(err))
# sys.exit(1)
3 changes: 1 addition & 2 deletions infrastructure-provisioning/src/edge/scripts/configure_nginx_reverse_proxy.py
View file
Expand Up @@ -56,8 +56,7 @@
install_nginx_ldap(args.hostname, os.environ['reverse_proxy_nginx_version'],
os.environ['ldap_hostname'], os.environ['ldap_dn'],
os.environ['ldap_ou'], os.environ['ldap_service_password'],
os.environ['ldap_service_username'], os.environ['{}_iam_user'.format(
os.environ['conf_cloud_provider'])])
os.environ['ldap_service_username'], os.environ['aws_iam_user'])
except Exception as err:
print("Failed install nginx reverse proxy: " + str(err))
sys.exit(1)
Expand Down
0 ...mplates/locations/dataengine-service.conf → ...ing/src/edge/templates/locations/emr.conf
View file
File renamed without changes.
8 changes: 1 addition & 7 deletions infrastructure-provisioning/src/edge/templates/nginx.conf
View file
Expand Up @@ -39,25 +39,19 @@ http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 100;
keepalive_timeout 65;
types_hash_max_size 2048;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
client_max_body_size 50M;

include /etc/nginx/mime.types;
default_type application/octet-stream;

auth_ldap_cache_enabled on;
auth_ldap_cache_expiration_time 10000;
auth_ldap_cache_size 1000;

ldap_server ldap1 {
url ldap://LDAP_IP:389/LDAP_DN?uid,mail?sub?(&(objectClass=posixAccount)(uid=LDAP_USERNAME));
binddn "LDAP_SERVICE_USERNAME,LDAP_DN";
binddn_passwd "LDAP_SERVICE_PASSWORD";
require valid_user;
request_timeout 30s;
}

include /etc/nginx/conf.d/*.conf;
Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/check_inactivity.py
View file
Expand Up @@ -50,10 +50,10 @@
reply['response']['result'] = {"error": "Failed to open result.json"}

reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/configure.py
View file
Expand Up @@ -49,10 +49,10 @@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/create.py
View file
Expand Up @@ -56,10 +56,10 @@
response_file.write(json.dumps(reply))
else:
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/create_image.py
View file
Expand Up @@ -50,10 +50,10 @@
reply['response']['result'] = {"error": "Failed to open result.json"}

reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/git_creds.py
View file
Expand Up @@ -50,10 +50,10 @@
reply['response']['result'] = {"error": "Failed to open result.json"}

reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down
4 changes: 2 additions & 2 deletions infrastructure-provisioning/src/general/api/install_libs.py
View file
Expand Up @@ -50,10 +50,10 @@
reply['response']['result'] = {"error": "Failed to open result.json"}

reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['edge_user_name'],
os.environ['project_name'],
os.environ['request_id'])

with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['edge_user_name'],
with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))

Expand Down

0 comments on commit 75c4b3b

Please sign in to comment.