Rename MaxCompute env variables to be consistent with SQLFlow (#1609)

* Rename MaxCompute env variables to be consistent with SQLFlow

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

* Fix test

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

* Fix env var name

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

.travis.yml

@@ -47,7 +47,7 @@ jobs:
           if [ "$ODPS_ACCESS_ID" == "" ] || [ "$ODPS_ACCESS_KEY" == "" ]; then
             echo "Skipping ODPS related unit tests since either ODPS_ACCESS_ID or ODPS_ACCESS_KEY is not set"
           else
-            docker run --rm -it -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
+            docker run --rm -it -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
           fi
         # Run integration tests
         - kubectl apply -f elasticdl/manifests/examples/elasticdl-rbac.yaml
@@ -66,17 +66,17 @@ jobs:
               else
                 echo "Running ElasticDL job: ${JOB_TYPE}"
                 if [[ "$JOB_TYPE" == "odps" ]]; then
-                    export ODPS_TABLE_NAME="odps_integration_build_${TRAVIS_BUILD_NUMBER}_$(date +%s)"
-                    docker run --rm -it -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; create_iris_odps_table_from_env()"'
+                    export MAXCOMPUTE_TABLE="odps_integration_build_${TRAVIS_BUILD_NUMBER}_$(date +%s)"
+                    docker run --rm -it -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; create_iris_odps_table_from_env()"'
                 fi
                 PS_NUM=2
                 WORKER_NUM=1
-                docker run --rm -it --net=host -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.kube:/root/.kube -v /home/$USER/.minikube/:/home/$USER/.minikube/ -v $(pwd):/work -w /work elasticdl:ci bash -c "scripts/client_test.sh ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}"
+                docker run --rm -it --net=host -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.kube:/root/.kube -v /home/$USER/.minikube/:/home/$USER/.minikube/ -v $(pwd):/work -w /work elasticdl:ci bash -c "scripts/client_test.sh ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}"
                 if [[ "$JOB_TYPE" != "local" ]]; then
                     python3 scripts/validate_job_status.py ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}
                 fi
                 if [[ "$JOB_TYPE" == "odps" ]]; then
-                    docker run --rm -it -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; delete_iris_odps_table_from_env()"'
+                    docker run --rm -it -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; delete_iris_odps_table_from_env()"'
                 fi
               fi
           done
@@ -92,4 +92,4 @@ env:
   # Secure tokens for ODPS-related environment variables
   - secure: kr79IltMhuW+pmmdP0KBMY87+iePGBGiaOUYTZhxx3zW3a1t2xK5wJnxJvrOwgBipqxLvIQnYUu20Lwfo0I2bwPRVvZAExb1vFBDQgzCaXEj+DHanE0XR1nivMzUcI3iHiBNRo5GalAUuiCzc/8fTxwcd0az8uxbWgPsTGkE0b8Y4epmLfsBn87rfc/lq6zWV9Q/dogBtiSZSS+bWR+U1/KopoY2hQE9hDVlZwlh/5gqbtWDOKDWE+pOnHJfHzfLzjrTb1qKgcUdA0FWU4+TyXwU09qYG5YvXRDHb6tpryacQC99E2rLkVZSqhIaSxjGAxbIcpvi0osv0R2FfeRd1hpgtB1Ro+NkZioW7dfZfhMxJm8Q0yVCmNx3D4HyZzGX/rO4estNZbX2+Zq522wakX7YCQ7TYWjWaJJNOOuYJaFp8y80sa+kE4ecF/5ZPSFiL/pUqbmnLNLNgYIq//jVgsyvJyW5luYdmD5+oTXKYi67ofzGqtY6y2nAJYHzkR2iVTwJ7CqxamkQ+3tPgoorApEc/I3DdLEIWrjTbHIfzQC/RpvdJukF3hjIrrcF/CnHq//rumVjPkn05yX4LzO/H5q6Bdto/+o+RGfamdAfJD09nfQAw7lQZ4yzUdzhSILTvONSNkLygyvW1NUL353TXXCa36uQvFwguJiSwzS2eDM=
   - secure: OHi/YUNWjG2NGRNPkTSULN7d7fJG4/uFGIpsSrh6MUl7zFIK39Qh1enCPeHmOKO5Fo6HxiiqzN0TbGwxnx5gkptEYwLV0DgWrT6pugwvWBehDU5koJmaEDuqJ5EGLHhJ0fm0wESHzv9hlK+Pb9vTD6by8X40LAwgU1PdcS9W5hpzXJuM2cRoRp8kiOKv7vJZXhbdYCcSU2zpizhJgh6B6zpv5gI0rdgTPL8EDYCxUQg1RrNw27+Dti3kCD59FphlRnxMzC7OoANJdFHMDBmruQIdq4/0UfPrFp7ZubwM3k+DFuZkRohyPeup8L0GYTwIt+PzuEC1+1rhIy7kxinwCn0jc/DK7hSEduxxr+CbPSBV60oqi+eZSahLFZYG26xsNUR5q7B2n36pYj0eK8d54ionHqsqP39ure10T+mRZS/SF9KPcpo+ZeEJaPTP41s3/5i3r5BlKfGr9GaA1yVtsbOwTx4McwJfW7Vd/HN+++swP+x1pPsC6iJ8KluZ6iSx6andHGnLyzvltaPqXxR8KNmfhDKXCcX8U/OPN1v22irdyAqWrZe7geNNVZJF1BRMEby1UYmAVZyZJ8xfhod/Acpv3Di+ZKSGGhbKvt20J1oYpVsV5lcNGAuCLgd175HHgkpFfjGuw97rRberBgCwzQdrRJjSvgsSp7VPjzOnrAE=
-  - ODPS_PROJECT_NAME=gomaxcompute_driver_w7u
+  - MAXCOMPUTE_PROJECT=gomaxcompute_driver_w7u

elasticdl/README.md

@@ -89,14 +89,14 @@ the following:
 make -f elasticdl/Makefile && pytest elasticdl/python/tests
 ```
 
-[ODPS](https://www.alibabacloud.com/product/maxcompute)-related tests require additional environment variables. To run those tests, execute the following:
+[MaxCompute](https://www.alibabacloud.com/product/maxcompute)-related tests require additional environment variables. To run those tests, execute the following:
 
 ```bash
 docker run --rm -it -v $PWD:/edl_dir -w /edl_dir \
-    -e ODPS_PROJECT_NAME=xxx \
-    -e ODPS_ACCESS_ID=xxx \
-    -e ODPS_ACCESS_KEY=xxx \
-    -e ODPS_ENDPOINT=xxx \
+    -e MAXCOMPUTE_PROJECT=xxx \
+    -e MAXCOMPUTE_AK=xxx \
+    -e MAXCOMPUTE_SK=xxx \
+    -e MAXCOMPUTE_ENDPOINT=xxx \
     elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
 ```
 
@@ -171,16 +171,16 @@ All tests will be executed on [Travis CI](https://travis-ci.org/sql-machine-lear
 * Integration tests
 
 The unit tests and integration tests also contain tests running on a local Kubernetes cluster via [Minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) and tests that
-require data sources from [ODPS](https://www.alibabacloud.com/product/maxcompute). Please refer to [Travis configuration file](../.travis.yml) for more details.
+require data sources from [MaxCompute](https://www.alibabacloud.com/product/maxcompute). Please refer to [Travis configuration file](../.travis.yml) for more details.
 
-Note that tests related to ODPS will not be executed on pull requests created from forks since
-the ODPS access information has been secured on Travis and only those who have write access can retrieve it. Developers who
-have write access to this repo are encouraged to submit pull requests from branches instead of forks if any code related to ODPS
+Note that tests related to MaxCompute will not be executed on pull requests created from forks since
+the MaxCompute access information has been secured on Travis and only those who have write access can retrieve it. Developers who
+have write access to this repo are encouraged to submit pull requests from branches instead of forks if any code related to MaxCompute
 has been modified.
 
 Also note that two test cases of integration tests involve loading checkpoint. It is not easy to automatically generate checkpoints when doing integration tests. Currently we save a checkpoint file in the [test data folder](python/tests/testdata) of the ElasticDL Github repository and use this checkpoint file for integration tests. Thus you need to re-generate a new checkpoint file if your PR modifies the definition of Model protocol buffer.
 
 If you want to trigger Travis builds without submitting a pull request, you can do so by developing on a branch and add this
 branch name to the list in `branches` section in [Travis configuration file](../.travis.yml). Note that you can also trigger
-Travis builds from forks but it requires additional work such as activating Travis for the forked repo and ODPS related tests
+Travis builds from forks but it requires additional work such as activating Travis for the forked repo and MaxCompute related tests
 will be skipped as mentioned earlier.

elasticdl/python/common/constants.py

@@ -11,11 +11,11 @@ class InstanceManagerStatus(object):
     FINISHED = "Finished"
 
 
-class ODPSConfig(object):
-    PROJECT_NAME = "ODPS_PROJECT_NAME"
-    ACCESS_ID = "ODPS_ACCESS_ID"
-    ACCESS_KEY = "ODPS_ACCESS_KEY"
-    ENDPOINT = "ODPS_ENDPOINT"
+class MaxComputeConfig(object):
+    PROJECT_NAME = "MAXCOMPUTE_PROJECT"
+    ACCESS_ID = "MAXCOMPUTE_AK"
+    ACCESS_KEY = "MAXCOMPUTE_SK"
+    ENDPOINT = "MAXCOMPUTE_ENDPOINT"
 
 
 class JobType(object):

elasticdl/python/data/odps_io.py

@@ -10,7 +10,7 @@
 from odps import ODPS
 from odps.models import Schema
 
-from elasticdl.python.common.constants import ODPSConfig
+from elasticdl.python.common.constants import MaxComputeConfig
 from elasticdl.python.common.log_utils import default_logger as logger
 
 
@@ -51,9 +51,9 @@ def is_odps_configured():
     return all(
         k in os.environ
         for k in (
-            ODPSConfig.PROJECT_NAME,
-            ODPSConfig.ACCESS_ID,
-            ODPSConfig.ACCESS_KEY,
+            MaxComputeConfig.PROJECT_NAME,
+            MaxComputeConfig.ACCESS_ID,
+            MaxComputeConfig.ACCESS_KEY,
         )
     )
 

elasticdl/python/data/reader/data_reader_factory.py

@@ -1,6 +1,6 @@
 import os
 
-from elasticdl.python.common.constants import ODPSConfig, ReaderType
+from elasticdl.python.common.constants import MaxComputeConfig, ReaderType
 from elasticdl.python.data.odps_io import is_odps_configured
 from elasticdl.python.data.reader.csv_reader import CSVDataReader
 from elasticdl.python.data.reader.odps_reader import ODPSDataReader
@@ -17,11 +17,11 @@ def create_data_reader(data_origin, records_per_task=None, **kwargs):
     if reader_type is None:
         if is_odps_configured():
             return ODPSDataReader(
-                project=os.environ[ODPSConfig.PROJECT_NAME],
-                access_id=os.environ[ODPSConfig.ACCESS_ID],
-                access_key=os.environ[ODPSConfig.ACCESS_KEY],
+                project=os.environ[MaxComputeConfig.PROJECT_NAME],
+                access_id=os.environ[MaxComputeConfig.ACCESS_ID],
+                access_key=os.environ[MaxComputeConfig.ACCESS_KEY],
                 table=data_origin,
-                endpoint=os.environ.get(ODPSConfig.ENDPOINT),
+                endpoint=os.environ.get(MaxComputeConfig.ENDPOINT),
                 records_per_task=records_per_task,
                 **kwargs,
             )
@@ -34,15 +34,15 @@ def create_data_reader(data_origin, records_per_task=None, **kwargs):
     elif reader_type == ReaderType.ODPS_READER:
         if not is_odps_configured:
             raise ValueError(
-                "ODPS_ACCESS_ID, ODPS_ACCESS_KEY and ODPS_PROJECT_NAME ",
+                "MAXCOMPUTE_AK, MAXCOMPUTE_SK and MAXCOMPUTE_PROJECT ",
                 "must be configured in envs",
             )
         return ODPSDataReader(
-            project=os.environ[ODPSConfig.PROJECT_NAME],
-            access_id=os.environ[ODPSConfig.ACCESS_ID],
-            access_key=os.environ[ODPSConfig.ACCESS_KEY],
+            project=os.environ[MaxComputeConfig.PROJECT_NAME],
+            access_id=os.environ[MaxComputeConfig.ACCESS_ID],
+            access_key=os.environ[MaxComputeConfig.ACCESS_KEY],
             table=data_origin,
-            endpoint=os.environ.get(ODPSConfig.ENDPOINT),
+            endpoint=os.environ.get(MaxComputeConfig.ENDPOINT),
             records_per_task=records_per_task,
             **kwargs,
         )

elasticdl/python/tests/data_reader_test.py

@@ -9,7 +9,7 @@
 import tensorflow as tf
 from odps import ODPS
 
-from elasticdl.python.common.constants import ODPSConfig
+from elasticdl.python.common.constants import MaxComputeConfig
 from elasticdl.python.common.model_utils import load_module
 from elasticdl.python.data.odps_io import is_odps_configured
 from elasticdl.python.data.reader.csv_reader import CSVDataReader
@@ -104,10 +104,10 @@ def _parse_data(record):
 )
 class ODPSDataReaderTest(unittest.TestCase):
     def setUp(self):
-        self.project = os.environ[ODPSConfig.PROJECT_NAME]
-        access_id = os.environ[ODPSConfig.ACCESS_ID]
-        access_key = os.environ[ODPSConfig.ACCESS_KEY]
-        endpoint = os.environ.get(ODPSConfig.ENDPOINT)
+        self.project = os.environ[MaxComputeConfig.PROJECT_NAME]
+        access_id = os.environ[MaxComputeConfig.ACCESS_ID]
+        access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
+        endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
         self.test_table = "test_odps_data_reader_%d_%d" % (
             int(time.time()),
             random.randint(1, 101),

elasticdl/python/tests/odps_io_test.py

@@ -6,7 +6,7 @@
 
 from odps import ODPS
 
-from elasticdl.python.common.constants import ODPSConfig
+from elasticdl.python.common.constants import MaxComputeConfig
 from elasticdl.python.data.odps_io import (
     ODPSReader,
     ODPSWriter,
@@ -23,10 +23,10 @@
 )
 class ODPSIOTest(unittest.TestCase):
     def setUp(self):
-        self._project = os.environ[ODPSConfig.PROJECT_NAME]
-        self._access_id = os.environ[ODPSConfig.ACCESS_ID]
-        self._access_key = os.environ[ODPSConfig.ACCESS_KEY]
-        self._endpoint = os.environ.get(ODPSConfig.ENDPOINT)
+        self._project = os.environ[MaxComputeConfig.PROJECT_NAME]
+        self._access_id = os.environ[MaxComputeConfig.ACCESS_ID]
+        self._access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
+        self._endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
         self._test_read_table = "test_odps_reader_%d_%d" % (
             int(time.time()),
             random.randint(1, 101),

elasticdl/python/tests/test_utils.py

@@ -16,7 +16,7 @@
 from elasticdl.python.common.constants import (
     DistributionStrategy,
     JobType,
-    ODPSConfig,
+    MaxComputeConfig,
 )
 from elasticdl.python.common.grpc_utils import build_channel
 from elasticdl.python.common.model_utils import (
@@ -510,22 +510,22 @@ class BIGINT);
 
 
 def get_odps_client_from_env():
-    project = os.environ[ODPSConfig.PROJECT_NAME]
-    access_id = os.environ[ODPSConfig.ACCESS_ID]
-    access_key = os.environ[ODPSConfig.ACCESS_KEY]
-    endpoint = os.environ.get(ODPSConfig.ENDPOINT)
+    project = os.environ[MaxComputeConfig.PROJECT_NAME]
+    access_id = os.environ[MaxComputeConfig.ACCESS_ID]
+    access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
+    endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
     return ODPS(access_id, access_key, project, endpoint)
 
 
 def create_iris_odps_table_from_env():
-    project = os.environ[ODPSConfig.PROJECT_NAME]
-    table_name = os.environ["ODPS_TABLE_NAME"]
+    project = os.environ[MaxComputeConfig.PROJECT_NAME]
+    table_name = os.environ["MAXCOMPUTE_TABLE"]
     create_iris_odps_table(get_odps_client_from_env(), project, table_name)
 
 
 def delete_iris_odps_table_from_env():
-    project = os.environ[ODPSConfig.PROJECT_NAME]
-    table_name = os.environ["ODPS_TABLE_NAME"]
+    project = os.environ[MaxComputeConfig.PROJECT_NAME]
+    table_name = os.environ["MAXCOMPUTE_TABLE"]
     get_odps_client_from_env().delete_table(
         table_name, project, if_exists=True
     )

model_zoo/cifar10_functional_api/cifar10_functional_api.py

@@ -2,7 +2,7 @@
 
 import tensorflow as tf
 
-from elasticdl.python.common.constants import Mode, ODPSConfig
+from elasticdl.python.common.constants import MaxComputeConfig, Mode
 from elasticdl.python.common.log_utils import default_logger as logger
 from elasticdl.python.data.odps_io import ODPSWriter, is_odps_configured
 from elasticdl.python.worker.prediction_outputs_processor import (
@@ -163,10 +163,10 @@ class PredictionOutputsProcessor(BasePredictionOutputsProcessor):
     def __init__(self):
         if is_odps_configured():
             self.odps_writer = ODPSWriter(
-                os.environ[ODPSConfig.PROJECT_NAME],
-                os.environ[ODPSConfig.ACCESS_ID],
-                os.environ[ODPSConfig.ACCESS_KEY],
-                os.environ[ODPSConfig.ENDPOINT],
+                os.environ[MaxComputeConfig.PROJECT_NAME],
+                os.environ[MaxComputeConfig.ACCESS_ID],
+                os.environ[MaxComputeConfig.ACCESS_KEY],
+                os.environ[MaxComputeConfig.ENDPOINT],
                 "cifar10_prediction_outputs",
                 # TODO: Print out helpful error message if the columns and
                 # column_types do not match with the prediction outputs

scripts/client_test.sh

@@ -93,9 +93,9 @@ elif [[ "$JOB_TYPE" == "odps" ]]; then
       --image_base=elasticdl:ci \
       --model_zoo=model_zoo \
       --model_def=odps_iris_dnn_model.odps_iris_dnn_model.custom_model \
-      --training_data=$ODPS_TABLE_NAME \
+      --training_data=$MAXCOMPUTE_TABLE \
       --data_reader_params='columns=["sepal_length", "sepal_width", "petal_length", "petal_width", "class"]; label_col="class"' \
-      --envs="ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME,ODPS_ACCESS_ID=$ODPS_ACCESS_ID,ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY,ODPS_ENDPOINT=" \
+      --envs="MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT,MAXCOMPUTE_AK=$MAXCOMPUTE_AK,MAXCOMPUTE_SK=$MAXCOMPUTE_SK,MAXCOMPUTE_ENDPOINT=" \
       --num_epochs=2 \
       --master_resource_request="cpu=0.2,memory=1024Mi" \
       --master_resource_limit="cpu=1,memory=2048Mi" \