Skip to content
Permalink
Browse files

Rename MaxCompute env variables to be consistent with SQLFlow (#1609)

* Rename MaxCompute env variables to be consistent with SQLFlow

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

* Fix test

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

* Fix env var name

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>
  • Loading branch information
terrytangyuan committed Jan 3, 2020
1 parent f1ab88d commit 3fd0b1227996bd56453805c5581e143423248239
@@ -47,7 +47,7 @@ jobs:
if [ "$ODPS_ACCESS_ID" == "" ] || [ "$ODPS_ACCESS_KEY" == "" ]; then
echo "Skipping ODPS related unit tests since either ODPS_ACCESS_ID or ODPS_ACCESS_KEY is not set"
else
docker run --rm -it -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
docker run --rm -it -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
fi
# Run integration tests
- kubectl apply -f elasticdl/manifests/examples/elasticdl-rbac.yaml
@@ -66,17 +66,17 @@ jobs:
else
echo "Running ElasticDL job: ${JOB_TYPE}"
if [[ "$JOB_TYPE" == "odps" ]]; then
export ODPS_TABLE_NAME="odps_integration_build_${TRAVIS_BUILD_NUMBER}_$(date +%s)"
docker run --rm -it -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; create_iris_odps_table_from_env()"'
export MAXCOMPUTE_TABLE="odps_integration_build_${TRAVIS_BUILD_NUMBER}_$(date +%s)"
docker run --rm -it -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; create_iris_odps_table_from_env()"'
fi
PS_NUM=2
WORKER_NUM=1
docker run --rm -it --net=host -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.kube:/root/.kube -v /home/$USER/.minikube/:/home/$USER/.minikube/ -v $(pwd):/work -w /work elasticdl:ci bash -c "scripts/client_test.sh ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}"
docker run --rm -it --net=host -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v /var/run/docker.sock:/var/run/docker.sock -v $HOME/.kube:/root/.kube -v /home/$USER/.minikube/:/home/$USER/.minikube/ -v $(pwd):/work -w /work elasticdl:ci bash -c "scripts/client_test.sh ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}"
if [[ "$JOB_TYPE" != "local" ]]; then
python3 scripts/validate_job_status.py ${JOB_TYPE} ${PS_NUM} ${WORKER_NUM}
fi
if [[ "$JOB_TYPE" == "odps" ]]; then
docker run --rm -it -e ODPS_TABLE_NAME=$ODPS_TABLE_NAME -e ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME -e ODPS_ACCESS_ID=$ODPS_ACCESS_ID -e ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; delete_iris_odps_table_from_env()"'
docker run --rm -it -e MAXCOMPUTE_TABLE=$MAXCOMPUTE_TABLE -e MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT -e MAXCOMPUTE_AK=$ODPS_ACCESS_ID -e MAXCOMPUTE_SK=$ODPS_ACCESS_KEY -v $PWD:/work -w /work elasticdl:dev bash -c 'python -c "from elasticdl.python.tests.test_utils import *; delete_iris_odps_table_from_env()"'
fi
fi
done
@@ -92,4 +92,4 @@ env:
# Secure tokens for ODPS-related environment variables
- secure: kr79IltMhuW+pmmdP0KBMY87+iePGBGiaOUYTZhxx3zW3a1t2xK5wJnxJvrOwgBipqxLvIQnYUu20Lwfo0I2bwPRVvZAExb1vFBDQgzCaXEj+DHanE0XR1nivMzUcI3iHiBNRo5GalAUuiCzc/8fTxwcd0az8uxbWgPsTGkE0b8Y4epmLfsBn87rfc/lq6zWV9Q/dogBtiSZSS+bWR+U1/KopoY2hQE9hDVlZwlh/5gqbtWDOKDWE+pOnHJfHzfLzjrTb1qKgcUdA0FWU4+TyXwU09qYG5YvXRDHb6tpryacQC99E2rLkVZSqhIaSxjGAxbIcpvi0osv0R2FfeRd1hpgtB1Ro+NkZioW7dfZfhMxJm8Q0yVCmNx3D4HyZzGX/rO4estNZbX2+Zq522wakX7YCQ7TYWjWaJJNOOuYJaFp8y80sa+kE4ecF/5ZPSFiL/pUqbmnLNLNgYIq//jVgsyvJyW5luYdmD5+oTXKYi67ofzGqtY6y2nAJYHzkR2iVTwJ7CqxamkQ+3tPgoorApEc/I3DdLEIWrjTbHIfzQC/RpvdJukF3hjIrrcF/CnHq//rumVjPkn05yX4LzO/H5q6Bdto/+o+RGfamdAfJD09nfQAw7lQZ4yzUdzhSILTvONSNkLygyvW1NUL353TXXCa36uQvFwguJiSwzS2eDM=
- secure: OHi/YUNWjG2NGRNPkTSULN7d7fJG4/uFGIpsSrh6MUl7zFIK39Qh1enCPeHmOKO5Fo6HxiiqzN0TbGwxnx5gkptEYwLV0DgWrT6pugwvWBehDU5koJmaEDuqJ5EGLHhJ0fm0wESHzv9hlK+Pb9vTD6by8X40LAwgU1PdcS9W5hpzXJuM2cRoRp8kiOKv7vJZXhbdYCcSU2zpizhJgh6B6zpv5gI0rdgTPL8EDYCxUQg1RrNw27+Dti3kCD59FphlRnxMzC7OoANJdFHMDBmruQIdq4/0UfPrFp7ZubwM3k+DFuZkRohyPeup8L0GYTwIt+PzuEC1+1rhIy7kxinwCn0jc/DK7hSEduxxr+CbPSBV60oqi+eZSahLFZYG26xsNUR5q7B2n36pYj0eK8d54ionHqsqP39ure10T+mRZS/SF9KPcpo+ZeEJaPTP41s3/5i3r5BlKfGr9GaA1yVtsbOwTx4McwJfW7Vd/HN+++swP+x1pPsC6iJ8KluZ6iSx6andHGnLyzvltaPqXxR8KNmfhDKXCcX8U/OPN1v22irdyAqWrZe7geNNVZJF1BRMEby1UYmAVZyZJ8xfhod/Acpv3Di+ZKSGGhbKvt20J1oYpVsV5lcNGAuCLgd175HHgkpFfjGuw97rRberBgCwzQdrRJjSvgsSp7VPjzOnrAE=
- ODPS_PROJECT_NAME=gomaxcompute_driver_w7u
- MAXCOMPUTE_PROJECT=gomaxcompute_driver_w7u
@@ -89,14 +89,14 @@ the following:
make -f elasticdl/Makefile && pytest elasticdl/python/tests
```

[ODPS](https://www.alibabacloud.com/product/maxcompute)-related tests require additional environment variables. To run those tests, execute the following:
[MaxCompute](https://www.alibabacloud.com/product/maxcompute)-related tests require additional environment variables. To run those tests, execute the following:

```bash
docker run --rm -it -v $PWD:/edl_dir -w /edl_dir \
-e ODPS_PROJECT_NAME=xxx \
-e ODPS_ACCESS_ID=xxx \
-e ODPS_ACCESS_KEY=xxx \
-e ODPS_ENDPOINT=xxx \
-e MAXCOMPUTE_PROJECT=xxx \
-e MAXCOMPUTE_AK=xxx \
-e MAXCOMPUTE_SK=xxx \
-e MAXCOMPUTE_ENDPOINT=xxx \
elasticdl:dev bash -c "make -f elasticdl/Makefile && K8S_TESTS=False pytest elasticdl/python/tests/odps_* elasticdl/python/tests/data_reader_test.py"
```

@@ -171,16 +171,16 @@ All tests will be executed on [Travis CI](https://travis-ci.org/sql-machine-lear
* Integration tests

The unit tests and integration tests also contain tests running on a local Kubernetes cluster via [Minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) and tests that
require data sources from [ODPS](https://www.alibabacloud.com/product/maxcompute). Please refer to [Travis configuration file](../.travis.yml) for more details.
require data sources from [MaxCompute](https://www.alibabacloud.com/product/maxcompute). Please refer to [Travis configuration file](../.travis.yml) for more details.

Note that tests related to ODPS will not be executed on pull requests created from forks since
the ODPS access information has been secured on Travis and only those who have write access can retrieve it. Developers who
have write access to this repo are encouraged to submit pull requests from branches instead of forks if any code related to ODPS
Note that tests related to MaxCompute will not be executed on pull requests created from forks since
the MaxCompute access information has been secured on Travis and only those who have write access can retrieve it. Developers who
have write access to this repo are encouraged to submit pull requests from branches instead of forks if any code related to MaxCompute
has been modified.

Also note that two test cases of integration tests involve loading checkpoint. It is not easy to automatically generate checkpoints when doing integration tests. Currently we save a checkpoint file in the [test data folder](python/tests/testdata) of the ElasticDL Github repository and use this checkpoint file for integration tests. Thus you need to re-generate a new checkpoint file if your PR modifies the definition of Model protocol buffer.

If you want to trigger Travis builds without submitting a pull request, you can do so by developing on a branch and add this
branch name to the list in `branches` section in [Travis configuration file](../.travis.yml). Note that you can also trigger
Travis builds from forks but it requires additional work such as activating Travis for the forked repo and ODPS related tests
Travis builds from forks but it requires additional work such as activating Travis for the forked repo and MaxCompute related tests
will be skipped as mentioned earlier.
@@ -11,11 +11,11 @@ class InstanceManagerStatus(object):
FINISHED = "Finished"


class ODPSConfig(object):
PROJECT_NAME = "ODPS_PROJECT_NAME"
ACCESS_ID = "ODPS_ACCESS_ID"
ACCESS_KEY = "ODPS_ACCESS_KEY"
ENDPOINT = "ODPS_ENDPOINT"
class MaxComputeConfig(object):
PROJECT_NAME = "MAXCOMPUTE_PROJECT"
ACCESS_ID = "MAXCOMPUTE_AK"
ACCESS_KEY = "MAXCOMPUTE_SK"
ENDPOINT = "MAXCOMPUTE_ENDPOINT"


class JobType(object):
@@ -10,7 +10,7 @@
from odps import ODPS
from odps.models import Schema

from elasticdl.python.common.constants import ODPSConfig
from elasticdl.python.common.constants import MaxComputeConfig
from elasticdl.python.common.log_utils import default_logger as logger


@@ -51,9 +51,9 @@ def is_odps_configured():
return all(
k in os.environ
for k in (
ODPSConfig.PROJECT_NAME,
ODPSConfig.ACCESS_ID,
ODPSConfig.ACCESS_KEY,
MaxComputeConfig.PROJECT_NAME,
MaxComputeConfig.ACCESS_ID,
MaxComputeConfig.ACCESS_KEY,
)
)

@@ -1,6 +1,6 @@
import os

from elasticdl.python.common.constants import ODPSConfig, ReaderType
from elasticdl.python.common.constants import MaxComputeConfig, ReaderType
from elasticdl.python.data.odps_io import is_odps_configured
from elasticdl.python.data.reader.csv_reader import CSVDataReader
from elasticdl.python.data.reader.odps_reader import ODPSDataReader
@@ -17,11 +17,11 @@ def create_data_reader(data_origin, records_per_task=None, **kwargs):
if reader_type is None:
if is_odps_configured():
return ODPSDataReader(
project=os.environ[ODPSConfig.PROJECT_NAME],
access_id=os.environ[ODPSConfig.ACCESS_ID],
access_key=os.environ[ODPSConfig.ACCESS_KEY],
project=os.environ[MaxComputeConfig.PROJECT_NAME],
access_id=os.environ[MaxComputeConfig.ACCESS_ID],
access_key=os.environ[MaxComputeConfig.ACCESS_KEY],
table=data_origin,
endpoint=os.environ.get(ODPSConfig.ENDPOINT),
endpoint=os.environ.get(MaxComputeConfig.ENDPOINT),
records_per_task=records_per_task,
**kwargs,
)
@@ -34,15 +34,15 @@ def create_data_reader(data_origin, records_per_task=None, **kwargs):
elif reader_type == ReaderType.ODPS_READER:
if not is_odps_configured:
raise ValueError(
"ODPS_ACCESS_ID, ODPS_ACCESS_KEY and ODPS_PROJECT_NAME ",
"MAXCOMPUTE_AK, MAXCOMPUTE_SK and MAXCOMPUTE_PROJECT ",
"must be configured in envs",
)
return ODPSDataReader(
project=os.environ[ODPSConfig.PROJECT_NAME],
access_id=os.environ[ODPSConfig.ACCESS_ID],
access_key=os.environ[ODPSConfig.ACCESS_KEY],
project=os.environ[MaxComputeConfig.PROJECT_NAME],
access_id=os.environ[MaxComputeConfig.ACCESS_ID],
access_key=os.environ[MaxComputeConfig.ACCESS_KEY],
table=data_origin,
endpoint=os.environ.get(ODPSConfig.ENDPOINT),
endpoint=os.environ.get(MaxComputeConfig.ENDPOINT),
records_per_task=records_per_task,
**kwargs,
)
@@ -9,7 +9,7 @@
import tensorflow as tf
from odps import ODPS

from elasticdl.python.common.constants import ODPSConfig
from elasticdl.python.common.constants import MaxComputeConfig
from elasticdl.python.common.model_utils import load_module
from elasticdl.python.data.odps_io import is_odps_configured
from elasticdl.python.data.reader.csv_reader import CSVDataReader
@@ -104,10 +104,10 @@ def _parse_data(record):
)
class ODPSDataReaderTest(unittest.TestCase):
def setUp(self):
self.project = os.environ[ODPSConfig.PROJECT_NAME]
access_id = os.environ[ODPSConfig.ACCESS_ID]
access_key = os.environ[ODPSConfig.ACCESS_KEY]
endpoint = os.environ.get(ODPSConfig.ENDPOINT)
self.project = os.environ[MaxComputeConfig.PROJECT_NAME]
access_id = os.environ[MaxComputeConfig.ACCESS_ID]
access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
self.test_table = "test_odps_data_reader_%d_%d" % (
int(time.time()),
random.randint(1, 101),
@@ -6,7 +6,7 @@

from odps import ODPS

from elasticdl.python.common.constants import ODPSConfig
from elasticdl.python.common.constants import MaxComputeConfig
from elasticdl.python.data.odps_io import (
ODPSReader,
ODPSWriter,
@@ -23,10 +23,10 @@
)
class ODPSIOTest(unittest.TestCase):
def setUp(self):
self._project = os.environ[ODPSConfig.PROJECT_NAME]
self._access_id = os.environ[ODPSConfig.ACCESS_ID]
self._access_key = os.environ[ODPSConfig.ACCESS_KEY]
self._endpoint = os.environ.get(ODPSConfig.ENDPOINT)
self._project = os.environ[MaxComputeConfig.PROJECT_NAME]
self._access_id = os.environ[MaxComputeConfig.ACCESS_ID]
self._access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
self._endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
self._test_read_table = "test_odps_reader_%d_%d" % (
int(time.time()),
random.randint(1, 101),
@@ -16,7 +16,7 @@
from elasticdl.python.common.constants import (
DistributionStrategy,
JobType,
ODPSConfig,
MaxComputeConfig,
)
from elasticdl.python.common.grpc_utils import build_channel
from elasticdl.python.common.model_utils import (
@@ -510,22 +510,22 @@ class BIGINT);


def get_odps_client_from_env():
project = os.environ[ODPSConfig.PROJECT_NAME]
access_id = os.environ[ODPSConfig.ACCESS_ID]
access_key = os.environ[ODPSConfig.ACCESS_KEY]
endpoint = os.environ.get(ODPSConfig.ENDPOINT)
project = os.environ[MaxComputeConfig.PROJECT_NAME]
access_id = os.environ[MaxComputeConfig.ACCESS_ID]
access_key = os.environ[MaxComputeConfig.ACCESS_KEY]
endpoint = os.environ.get(MaxComputeConfig.ENDPOINT)
return ODPS(access_id, access_key, project, endpoint)


def create_iris_odps_table_from_env():
project = os.environ[ODPSConfig.PROJECT_NAME]
table_name = os.environ["ODPS_TABLE_NAME"]
project = os.environ[MaxComputeConfig.PROJECT_NAME]
table_name = os.environ["MAXCOMPUTE_TABLE"]
create_iris_odps_table(get_odps_client_from_env(), project, table_name)


def delete_iris_odps_table_from_env():
project = os.environ[ODPSConfig.PROJECT_NAME]
table_name = os.environ["ODPS_TABLE_NAME"]
project = os.environ[MaxComputeConfig.PROJECT_NAME]
table_name = os.environ["MAXCOMPUTE_TABLE"]
get_odps_client_from_env().delete_table(
table_name, project, if_exists=True
)
@@ -2,7 +2,7 @@

import tensorflow as tf

from elasticdl.python.common.constants import Mode, ODPSConfig
from elasticdl.python.common.constants import MaxComputeConfig, Mode
from elasticdl.python.common.log_utils import default_logger as logger
from elasticdl.python.data.odps_io import ODPSWriter, is_odps_configured
from elasticdl.python.worker.prediction_outputs_processor import (
@@ -163,10 +163,10 @@ class PredictionOutputsProcessor(BasePredictionOutputsProcessor):
def __init__(self):
if is_odps_configured():
self.odps_writer = ODPSWriter(
os.environ[ODPSConfig.PROJECT_NAME],
os.environ[ODPSConfig.ACCESS_ID],
os.environ[ODPSConfig.ACCESS_KEY],
os.environ[ODPSConfig.ENDPOINT],
os.environ[MaxComputeConfig.PROJECT_NAME],
os.environ[MaxComputeConfig.ACCESS_ID],
os.environ[MaxComputeConfig.ACCESS_KEY],
os.environ[MaxComputeConfig.ENDPOINT],
"cifar10_prediction_outputs",
# TODO: Print out helpful error message if the columns and
# column_types do not match with the prediction outputs
@@ -93,9 +93,9 @@ elif [[ "$JOB_TYPE" == "odps" ]]; then
--image_base=elasticdl:ci \
--model_zoo=model_zoo \
--model_def=odps_iris_dnn_model.odps_iris_dnn_model.custom_model \
--training_data=$ODPS_TABLE_NAME \
--training_data=$MAXCOMPUTE_TABLE \
--data_reader_params='columns=["sepal_length", "sepal_width", "petal_length", "petal_width", "class"]; label_col="class"' \
--envs="ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME,ODPS_ACCESS_ID=$ODPS_ACCESS_ID,ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY,ODPS_ENDPOINT=" \
--envs="MAXCOMPUTE_PROJECT=$MAXCOMPUTE_PROJECT,MAXCOMPUTE_AK=$MAXCOMPUTE_AK,MAXCOMPUTE_SK=$MAXCOMPUTE_SK,MAXCOMPUTE_ENDPOINT=" \
--num_epochs=2 \
--master_resource_request="cpu=0.2,memory=1024Mi" \
--master_resource_limit="cpu=1,memory=2048Mi" \

0 comments on commit 3fd0b12

Please sign in to comment.
You can’t perform that action at this time.