Make hsds a Python package

Dockerfile

@@ -1,12 +1,11 @@
 FROM hdfgroup/python:3.8
 MAINTAINER John Readey <jreadey@hdfgroup.org>
-
-RUN mkdir /usr/local/src/hsds/ /usr/local/src/tests/
-COPY hsds /usr/local/src/hsds/
-COPY tests /usr/local/src/tests/
-COPY testall.py /usr/local/src/
+RUN mkdir /usr/local/src/hsds-src/ /usr/local/src/hsds/
+COPY . /usr/local/src/hsds-src
+RUN pip install /usr/local/src/hsds-src/[azure]
+COPY admin/config/passwd.txt /usr/local/src/hsds/
 COPY entrypoint.sh  /
 
 EXPOSE 5100-5999
-
-ENTRYPOINT ["/entrypoint.sh"]
+ 
+ENTRYPOINT ["/entrypoint.sh"]

admin/systemd/run.sh

@@ -25,14 +25,17 @@ echo "cwd:" ${PWD}
 if [ $1 == "head" ]; then
     # magic url which returns local ip when run on EC2 host
     export HEAD_HOST=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
-    echo "running headnode.py"
-    python -u headnode.py &
+    echo "running hsds-headnode"
+    export PYTHONUNBUFFERED="1"
+    hsds-headnode &
 elif [ $1 == "dn" ]; then
-    echo "running datanode.py"
-    python -u datanode.py &
+    echo "running hsds-datanode"
+    export PYTHONUNBUFFERED="1"
+    hsds-datanode &
 elif [ $1 == "sn" ]; then
-    echo "running servicenode.py"
-    python -u servicenode.py &
+    echo "running hsds-servicenode"
+    export PYTHONUNBUFFERED="1"
+    hsds-servicenode &
 else
     echo "Argument not recognized"
     exit 1

entrypoint.sh

@@ -1,29 +1,28 @@
 #!/bin/bash
 echo "hsds entrypoint"
 echo "node type: " $NODE_TYPE
-cd /usr/local/src/hsds
+cd /usr/local/src/
 
 
 if [ -z $NODE_TYPE ]; then
   # run client
-  cd /usr/local/src/
   if [ -z "$RUN_COMMAND" ]; then
     /bin/bash
   else
     /bin/bash -c "$RUN_COMMAND"
   fi
 elif [ $NODE_TYPE == "dn" ]; then
-  echo "running datanode.py"
-  python -u datanode.py 
+  echo "running hsds-datanode"
+  export PYTHONUNBUFFERED="1"
+  hsds-datanode
 elif [ $NODE_TYPE == "sn" ]; then
-  echo "running servicenode.py"
-  python -u servicenode.py 
-elif [ $NODE_TYPE == "an" ]; then
-  echo "running asyncnode.py"
-  python -u asyncnode.py
+  echo "running hsds-servicenode"
+  export PYTHONUNBUFFERED="1"
+  hsds-servicenode
 elif [ $NODE_TYPE == "head_node" ]; then
-  echo "running headnode.py"
-  python -u headnode.py 
+  echo "running hsds-headnode"
+  export PYTHONUNBUFFERED="1"
+  hsds-headnode
 else
   echo "Unknown NODE_TYPE: " $NODE_TYPE
 fi

hsds/async_lib.py

@@ -13,11 +13,11 @@
 import time
 from aiohttp.client_exceptions import ClientError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError
-from util.idUtil import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey, getObjId, isValidChunkId, getCollectionForId
-from util.chunkUtil import getDatasetId
-from util.storUtil import getStorKeys, putStorJSONObj, deleteStorObj
-import hsds_logger as log
-import config
+from .util.idUtil import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey, getObjId, isValidChunkId, getCollectionForId
+from .util.chunkUtil import getDatasetId
+from .util.storUtil import getStorKeys, putStorJSONObj, deleteStorObj
+from . import hsds_logger as log
+from . import config
 
 
 # List all keys under given root and optionally update info.json

hsds/attr_dn.py

@@ -18,9 +18,9 @@
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPConflict, HTTPNotFound, HTTPInternalServerError
 from aiohttp.web import json_response
 
-from util.attrUtil import validateAttributeName
-from datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj
-import hsds_logger as log
+from .util.attrUtil import validateAttributeName
+from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj
+from . import hsds_logger as log
 
 def index(a, x):
     """ Locate the leftmost value exactly equal to x

hsds/attr_sn.py

@@ -17,15 +17,15 @@
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
 from aiohttp.web import StreamResponse
 
-from util.httpUtil import  http_get, http_put, http_delete, getHref, getAcceptType, jsonResponse
-from util.idUtil import   isValidUuid, getDataNodeUrl
-from util.authUtil import getUserPasswordFromRequest, validateUserPassword
-from util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
-from util.attrUtil import  validateAttributeName, getRequestCollectionName
-from util.hdf5dtype import validateTypeItem, getBaseTypeJson, createDataType, getItemSize
-from util.arrayUtil import jsonToArray, getShapeDims, getNumElements, bytesArrayToList
-from servicenode_lib import getDomainJson, getObjectJson, validateAction
-import hsds_logger as log
+from .util.httpUtil import  http_get, http_put, http_delete, getHref, getAcceptType, jsonResponse
+from .util.idUtil import   isValidUuid, getDataNodeUrl
+from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
+from .util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
+from .util.attrUtil import  validateAttributeName, getRequestCollectionName
+from .util.hdf5dtype import validateTypeItem, getBaseTypeJson, createDataType, getItemSize
+from .util.arrayUtil import jsonToArray, getShapeDims, getNumElements, bytesArrayToList
+from .servicenode_lib import getDomainJson, getObjectJson, validateAction
+from . import hsds_logger as log
 
 
 async def GET_Attributes(request):

hsds/basenode.py

@@ -27,12 +27,12 @@
 from asyncio import CancelledError
 
 
-import config
-from util.httpUtil import http_get, http_post, jsonResponse
-from util.idUtil import createNodeId
-from util.authUtil import getUserPasswordFromRequest, validateUserPassword
-import util.query_marathon as marathonClient
-import hsds_logger as log
+from . import config
+from .util.httpUtil import http_get, http_post, jsonResponse
+from .util.idUtil import createNodeId
+from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
+from .util import query_marathon as marathonClient
+from . import hsds_logger as log
 from kubernetes import client as k8s_client
 from kubernetes import config as k8s_config
 

hsds/chunk_dn.py

@@ -18,18 +18,18 @@
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError, HTTPNotFound
 from aiohttp.web import json_response, StreamResponse
 
-from util.httpUtil import  request_read
-from util.arrayUtil import bytesToArray, arrayToBytes
-from util.idUtil import getS3Key, validateInPartition, isValidUuid
-from util.storUtil import  isStorObj, deleteStorObj
-from util.hdf5dtype import createDataType
-from util.dsetUtil import  getSliceQueryParam, getChunkLayout, getSelectionShape
-from util.chunkUtil import getChunkIndex, getDatasetId, chunkQuery
-from util.chunkUtil import chunkWriteSelection, chunkReadSelection
-from util.chunkUtil import chunkWritePoints, chunkReadPoints
-from datanode_lib import get_metadata_obj, get_chunk, save_chunk
-
-import hsds_logger as log
+from .util.httpUtil import  request_read
+from .util.arrayUtil import bytesToArray, arrayToBytes
+from .util.idUtil import getS3Key, validateInPartition, isValidUuid
+from .util.storUtil import  isStorObj, deleteStorObj
+from .util.hdf5dtype import createDataType
+from .util.dsetUtil import  getSliceQueryParam, getChunkLayout, getSelectionShape
+from .util.chunkUtil import getChunkIndex, getDatasetId, chunkQuery
+from .util.chunkUtil import chunkWriteSelection, chunkReadSelection
+from .util.chunkUtil import chunkWritePoints, chunkReadPoints
+from .datanode_lib import get_metadata_obj, get_chunk, save_chunk
+
+from . import hsds_logger as log
 
 """
 Update the requested chunk/selection

hsds/chunk_sn.py

@@ -23,20 +23,20 @@
 from aiohttp.client_exceptions import ClientError
 from aiohttp.web import StreamResponse
 
-from util.httpUtil import  getHref, getAcceptType, get_http_client, http_put, request_read, jsonResponse
-from util.idUtil import   isValidUuid, getDataNodeUrl
-from util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
-from util.hdf5dtype import getItemSize, createDataType
-from util.dsetUtil import getSliceQueryParam, setSliceQueryParam, getFillValue, isExtensible
-from util.dsetUtil import getSelectionShape, getDsetMaxDims, getChunkLayout, getDeflateLevel
-from util.chunkUtil import getNumChunks, getChunkIds, getChunkId, getChunkIndex, getChunkSuffix
-from util.chunkUtil import getChunkCoverage, getDataCoverage, getChunkIdForPartition
-from util.arrayUtil import bytesArrayToList, jsonToArray, getShapeDims, getNumElements, arrayToBytes, bytesToArray
-from util.authUtil import getUserPasswordFromRequest, validateUserPassword
-from util.awsLambdaClient import getLambdaClient
-from servicenode_lib import getObjectJson, validateAction
-import config
-import hsds_logger as log
+from .util.httpUtil import  getHref, getAcceptType, get_http_client, http_put, request_read, jsonResponse
+from .util.idUtil import   isValidUuid, getDataNodeUrl
+from .util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
+from .util.hdf5dtype import getItemSize, createDataType
+from .util.dsetUtil import getSliceQueryParam, setSliceQueryParam, getFillValue, isExtensible
+from .util.dsetUtil import getSelectionShape, getDsetMaxDims, getChunkLayout, getDeflateLevel
+from .util.chunkUtil import getNumChunks, getChunkIds, getChunkId, getChunkIndex, getChunkSuffix
+from .util.chunkUtil import getChunkCoverage, getDataCoverage, getChunkIdForPartition
+from .util.arrayUtil import bytesArrayToList, jsonToArray, getShapeDims, getNumElements, arrayToBytes, bytesToArray
+from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
+from .util.awsLambdaClient import getLambdaClient
+from .servicenode_lib import getObjectJson, validateAction
+from . import config
+from . import hsds_logger as log
 
 
 

hsds/ctype_dn.py

@@ -18,9 +18,9 @@
 from aiohttp.web import json_response
 
 
-from util.idUtil import isValidUuid, validateUuid
-from datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj, delete_metadata_obj, check_metadata_obj
-import hsds_logger as log
+from .util.idUtil import isValidUuid, validateUuid
+from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj, delete_metadata_obj, check_metadata_obj
+from . import hsds_logger as log
 
 
 async def GET_Datatype(request):

hsds/ctype_sn.py

@@ -17,13 +17,13 @@
 import json
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPGone
 
-from util.httpUtil import http_post, http_put, http_delete, getHref, jsonResponse
-from util.idUtil import   isValidUuid, getDataNodeUrl, createObjId
-from util.authUtil import getUserPasswordFromRequest, aclCheck, validateUserPassword
-from util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
-from util.hdf5dtype import validateTypeItem, getBaseTypeJson
-from servicenode_lib import getDomainJson, getObjectJson, validateAction, getObjectIdByPath, getPathForObjectId
-import hsds_logger as log
+from .util.httpUtil import http_post, http_put, http_delete, getHref, jsonResponse
+from .util.idUtil import   isValidUuid, getDataNodeUrl, createObjId
+from .util.authUtil import getUserPasswordFromRequest, aclCheck, validateUserPassword
+from .util.domainUtil import  getDomainFromRequest, isValidDomain, getBucketForDomain
+from .util.hdf5dtype import validateTypeItem, getBaseTypeJson
+from .servicenode_lib import getDomainJson, getObjectJson, validateAction, getObjectIdByPath, getPathForObjectId
+from . import hsds_logger as log
 
 
 async def GET_Datatype(request):

hsds/datanode.py

@@ -15,20 +15,20 @@
 import asyncio
 
 from aiohttp.web import run_app
-import config
-from util.lruCache import LruCache
-from util.idUtil import isValidUuid, isSchema2Id, getCollectionForId, isRootObjId
-from basenode import healthCheck, baseInit, preStop
-import hsds_logger as log
-from domain_dn import GET_Domain, PUT_Domain, DELETE_Domain, PUT_ACL
-from group_dn import GET_Group, POST_Group, DELETE_Group, PUT_Group, POST_Root
-from link_dn import GET_Links, GET_Link, PUT_Link, DELETE_Link
-from attr_dn import GET_Attributes, GET_Attribute, PUT_Attribute, DELETE_Attribute
-from ctype_dn import GET_Datatype, POST_Datatype, DELETE_Datatype
-from dset_dn import GET_Dataset, POST_Dataset, DELETE_Dataset, PUT_DatasetShape
-from chunk_dn import PUT_Chunk, GET_Chunk, POST_Chunk, DELETE_Chunk
-from datanode_lib import s3syncCheck
-from async_lib import scanRoot, removeKeys
+from . import config
+from .util.lruCache import LruCache
+from .util.idUtil import isValidUuid, isSchema2Id, getCollectionForId, isRootObjId
+from .basenode import healthCheck, baseInit, preStop
+from . import hsds_logger as log
+from .domain_dn import GET_Domain, PUT_Domain, DELETE_Domain, PUT_ACL
+from .group_dn import GET_Group, POST_Group, DELETE_Group, PUT_Group, POST_Root
+from .link_dn import GET_Links, GET_Link, PUT_Link, DELETE_Link
+from .attr_dn import GET_Attributes, GET_Attribute, PUT_Attribute, DELETE_Attribute
+from .ctype_dn import GET_Datatype, POST_Datatype, DELETE_Datatype
+from .dset_dn import GET_Dataset, POST_Dataset, DELETE_Dataset, PUT_DatasetShape
+from .chunk_dn import PUT_Chunk, GET_Chunk, POST_Chunk, DELETE_Chunk
+from .datanode_lib import s3syncCheck
+from .async_lib import scanRoot, removeKeys
 from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError, HTTPForbidden, HTTPBadRequest
 
 

hsds/datanode_lib.py

@@ -16,18 +16,18 @@
 import time
 import numpy as np
 from aiohttp.web_exceptions import HTTPGone, HTTPInternalServerError, HTTPBadRequest, HTTPNotFound, HTTPForbidden, HTTPServiceUnavailable
-from util.idUtil import validateInPartition, getS3Key, isValidUuid, isValidChunkId, getDataNodeUrl, isSchema2Id, getRootObjId, isRootObjId
-from util.storUtil import getStorJSONObj, putStorJSONObj, putStorBytes, getStorBytes, isStorObj, deleteStorObj
-from util.domainUtil import isValidDomain, getBucketForDomain
-from util.attrUtil import getRequestCollectionName
-from util.httpUtil import http_post
-from util.dsetUtil import getChunkLayout, getDeflateLevel, isShuffle, getFillValue
-from util.chunkUtil import getDatasetId
-from util.arrayUtil import arrayToBytes, bytesToArray
-from util.hdf5dtype import createDataType
-
-import config
-import hsds_logger as log
+from .util.idUtil import validateInPartition, getS3Key, isValidUuid, isValidChunkId, getDataNodeUrl, isSchema2Id, getRootObjId, isRootObjId
+from .util.storUtil import getStorJSONObj, putStorJSONObj, putStorBytes, getStorBytes, isStorObj, deleteStorObj
+from .util.domainUtil import isValidDomain, getBucketForDomain
+from .util.attrUtil import getRequestCollectionName
+from .util.httpUtil import http_post
+from .util.dsetUtil import getChunkLayout, getDeflateLevel, isShuffle, getFillValue
+from .util.chunkUtil import getDatasetId
+from .util.arrayUtil import arrayToBytes, bytesToArray
+from .util.hdf5dtype import createDataType
+
+from . import config
+from . import hsds_logger as log
 
 
 def get_obj_id(request, body=None):

hsds/domain_dn.py

@@ -16,11 +16,11 @@
 from aiohttp.web_exceptions import HTTPConflict, HTTPInternalServerError
 from aiohttp.web import json_response
 
-from util.authUtil import  getAclKeys
-from util.domainUtil import isValidDomain, getBucketForDomain
-from util.idUtil import validateInPartition
-from datanode_lib import get_metadata_obj, save_metadata_obj, delete_metadata_obj, check_metadata_obj
-import hsds_logger as log
+from .util.authUtil import  getAclKeys
+from .util.domainUtil import isValidDomain, getBucketForDomain
+from .util.idUtil import validateInPartition
+from .datanode_lib import get_metadata_obj, save_metadata_obj, delete_metadata_obj, check_metadata_obj
+from . import hsds_logger as log
 
 def get_domain(request, body=None):
     """ Extract domain and validate """

hsds/domain_sn.py

@@ -23,17 +23,17 @@
 from aiohttp import ClientResponseError
 from aiohttp.client_exceptions import ClientError
 
-from util.httpUtil import  http_post, http_put, http_get, http_delete, getHref, get_http_client, jsonResponse
-from util.idUtil import  getDataNodeUrl, createObjId, getCollectionForId, getDataNodeUrls
-from util.authUtil import getUserPasswordFromRequest, aclCheck
-from util.authUtil import validateUserPassword, getAclKeys
-from util.domainUtil import getParentDomain, getDomainFromRequest, isValidDomain, getBucketForDomain, getPathForDomain
-from util.storUtil import getStorKeys
-from util.boolparser import BooleanParser
-from servicenode_lib import getDomainJson, getObjectJson, getObjectIdByPath, getRootInfo
-from basenode import getVersion
-import hsds_logger as log
-import config
+from .util.httpUtil import  http_post, http_put, http_get, http_delete, getHref, get_http_client, jsonResponse
+from .util.idUtil import  getDataNodeUrl, createObjId, getCollectionForId, getDataNodeUrls
+from .util.authUtil import getUserPasswordFromRequest, aclCheck
+from .util.authUtil import validateUserPassword, getAclKeys
+from .util.domainUtil import getParentDomain, getDomainFromRequest, isValidDomain, getBucketForDomain, getPathForDomain
+from .util.storUtil import getStorKeys
+from .util.boolparser import BooleanParser
+from .servicenode_lib import getDomainJson, getObjectJson, getObjectIdByPath, getRootInfo
+from .basenode import getVersion
+from . import hsds_logger as log
+from . import config
 
 class DomainCrawler:
     def __init__(self, app, root_id, bucket=None, include_attrs=True, max_tasks=40):

hsds/dset_dn.py

@@ -18,9 +18,9 @@
 from aiohttp.web import json_response
 
 
-from util.idUtil import isValidUuid, validateUuid
-from datanode_lib import get_obj_id, check_metadata_obj, get_metadata_obj, save_metadata_obj, delete_metadata_obj
-import hsds_logger as log
+from .util.idUtil import isValidUuid, validateUuid
+from .datanode_lib import get_obj_id, check_metadata_obj, get_metadata_obj, save_metadata_obj, delete_metadata_obj
+from . import hsds_logger as log
 
 
 async def GET_Dataset(request):