Skip to content
This repository has been archived by the owner on Aug 17, 2023. It is now read-only.

Commit

Permalink
make fairing as a subpackage in kubeflow
Browse files Browse the repository at this point in the history
  • Loading branch information
jinchihe committed Sep 2, 2019
1 parent cd8b460 commit 2fc55d2
Show file tree
Hide file tree
Showing 144 changed files with 311 additions and 281 deletions.
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/bug_report.md
Expand Up @@ -19,7 +19,7 @@ about: Tell us about a problem you are experiencing

**Environment:**

- Fairing version: (use `python -c "import fairing; print(fairing.__version__)"`):
- Fairing version: (use `python -c "import kubeflow; print(kubeflow.__version__)"`):
- Kubeflow version: (version number can be found at the bottom left corner of the Kubeflow dashboard):
- Minikube version:
- Kubernetes version: (use `kubectl version`):
Expand Down
2 changes: 1 addition & 1 deletion .travis.yml
Expand Up @@ -9,5 +9,5 @@ install:
- pip install --upgrade pytest
# Command to run tests
script:
- cd tests/unit
- cd tests/fairing/unit
- pytest
4 changes: 2 additions & 2 deletions Dockerfile
Expand Up @@ -40,8 +40,8 @@ RUN pip install --user -r requirements.txt
COPY --chown=jovyan:users integration/ipython_kernel_config.py /home/jovyan/.ipython/profile_default/ipython_kernel_config.py
COPY --chown=jovyan:users containerregistry /home/jovyan/work/fairing/containerregistry

COPY --chown=jovyan:users examples/notebook /home/jovyan/work/fairing/
COPY --chown=jovyan:users fairing /home/jovyan/work/fairing/fairing
COPY --chown=jovyan:users examples/fairing/notebook /home/jovyan/work/fairing/
COPY --chown=jovyan:users kubeflow/fairing /home/jovyan/work/fairing/fairing



4 changes: 2 additions & 2 deletions Dockerfile.test
Expand Up @@ -17,8 +17,8 @@ ENV PATH=/builder/google-cloud-sdk/bin/:$PATH
RUN pip install pytest pytest-cov pytest-xdist papermill
# TODO: Find a better to run e2e tests without
# adding a lot of reqs
COPY examples/prediction/requirements.txt examples/prediction/requirements.txt
RUN pip install -r examples/prediction/requirements.txt
COPY examples/fairing/prediction/requirements.txt examples/fairing/prediction/requirements.txt
RUN pip install -r examples/fairing/prediction/requirements.txt

# Install dependencies
COPY setup.py requirements.txt /opt/fairing/
Expand Down
2 changes: 1 addition & 1 deletion cloudbuild.yaml
Expand Up @@ -19,7 +19,7 @@ steps:
entrypoint: 'bash'
args:
- '-c'
- 'gcloud container clusters get-credentials kubeflow-ci-fairing --zone us-central1-a && pytest -n 8 -v --durations=10 --cov=fairing --ignore=tests/integration/azure/ tests/'
- 'gcloud container clusters get-credentials kubeflow-ci-fairing --zone us-central1-a && pytest -n 8 -v --durations=10 --cov=fairing --ignore=tests/fairing/integration/azure/ tests/'

images: ['gcr.io/$PROJECT_ID/fairing-test:latest']
timeout: 3600s
Expand Down
Expand Up @@ -227,7 +227,7 @@ def feed_dict(train):

if __name__ == '__main__':
if os.getenv('FAIRING_RUNTIME', None) is None:
import fairing
from kubeflow import fairing
fairing.config.set_preprocessor('python', input_files=[__file__])
fairing.config.set_builder(
name='docker', registry='gcr.io/mrick-gcp', base_image='tensorflow/tensorflow')
Expand Down
Expand Up @@ -17,7 +17,7 @@ def train():
if os.getenv('FAIRING_RUNTIME', None) is not None:
train()
else:
import fairing
from kubeflow import fairing
# Setting up google container repositories (GCR) for storing output containers
# You can use any docker container registry istead of GCR
GCP_PROJECT = fairing.cloud.gcp.guess_project_name()
Expand Down
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.


import fairing
from kubeflow import fairing

if __name__ == '__main__':
fairing.config.set_deployer(
Expand Down
Expand Up @@ -73,7 +73,7 @@ def train():

if __name__ == '__main__':
if os.getenv('FAIRING_RUNTIME', None) is None:
import fairing
from kubeflow import fairing
fairing.config.set_preprocessor('python', input_files=[__file__])
fairing.config.set_builder(
'append', base_image='tensorflow/tensorflow:1.13.1-py3')
Expand Down
Expand Up @@ -74,7 +74,7 @@ def train(self, **kwargs): #pylint:disable=unused-argument

if __name__ == '__main__':
if os.getenv('FAIRING_RUNTIME', None) is None:
import fairing
from kubeflow import fairing
fairing.config.set_preprocessor('python', input_files=[__file__])
fairing.config.set_builder(name='docker', registry='<your-registry-name>',
base_image='tensorflow/tensorflow:1.13.1-py3')
Expand Down
Expand Up @@ -73,7 +73,7 @@ def train(self):

if __name__ == '__main__':
if os.getenv('FAIRING_RUNTIME', None) is None:
import fairing
from kubeflow import fairing
fairing.config.set_preprocessor('python', input_files=[__file__])
fairing.config.set_builder(
name='cluster', registry='<your-registry-here>')
Expand Down
Expand Up @@ -72,7 +72,7 @@ def train():

if __name__ == '__main__':
if os.getenv('FAIRING_RUNTIME', None) is None:
import fairing
from kubeflow import fairing
fairing.config.set_preprocessor('python', input_files=[__file__])
fairing.config.set_builder('docker', registry='<your-registry-name>',
base_image='tensorflow/tensorflow:1.13.1-py3')
Expand Down
File renamed without changes.
Expand Up @@ -15,8 +15,8 @@
"source": [
"import os\n",
"from time import gmtime, strftime\n",
"import fairing\n",
"from fairing.frameworks import lightgbm\n",
"from kubeflow import fairing\n",
"from kubeflow.fairing.frameworks import lightgbm\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
"# You can use any docker container registry istead of GCR\n",
Expand Down
Expand Up @@ -15,8 +15,8 @@
"source": [
"import os\n",
"from time import gmtime, strftime\n",
"import fairing\n",
"from fairing.frameworks import lightgbm\n",
"from kubeflow import fairing\n",
"from kubeflow.fairing.frameworks import lightgbm\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
"# You can use any docker container registry istead of GCR\n",
Expand Down
Expand Up @@ -13,7 +13,7 @@
"metadata": {},
"outputs": [],
"source": [
"import fairing\n",
"from kubeflow import fairing\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
"# You can use any docker container registry istead of GCR\n",
Expand Down Expand Up @@ -54,8 +54,8 @@
"metadata": {},
"outputs": [],
"source": [
"import fairing\n",
"from fairing.frameworks import lightgbm"
"from kubeflow import fairing\n",
"from kubeflow.fairing.frameworks import lightgbm"
]
},
{
Expand Down
Expand Up @@ -183,7 +183,7 @@
"metadata": {},
"outputs": [],
"source": [
"from fairing.deployers.gcp.gcpserving import GCPServingDeployer\n",
"from kubeflow.fairing.deployers.gcp.gcpserving import GCPServingDeployer\n",
"deployer = GCPServingDeployer()\n",
"deployer.deploy(VERSION_DIR, MODEL_NAME, VERSION_NAME)"
]
Expand Down
Expand Up @@ -34,7 +34,7 @@
"from tensorflow.examples.tutorials.mnist import input_data\n",
"from tensorflow.examples.tutorials.mnist import mnist\n",
"\n",
"import fairing\n",
"from kubeflow import fairing\n",
"fairing.config.set_builder(name='append')\n",
"\n",
"INPUT_DATA_DIR = '/tmp/tensorflow/mnist/input_data/'\n",
Expand Down
Expand Up @@ -114,7 +114,7 @@
"metadata": {},
"outputs": [],
"source": [
"import fairing\n",
"from kubeflow import fairing\n",
"run_training_and_eval = fairing.config.fn(run_training_and_eval)\n",
"run_training_and_eval()"
]
Expand Down
Expand Up @@ -7,7 +7,7 @@
"outputs": [],
"source": [
"import os\n",
"import fairing"
"from kubeflow import fairing"
]
},
{
Expand Down
File renamed without changes.
Expand Up @@ -13,8 +13,8 @@
# limitations under the License.

import joblib
import fairing
import numpy as np
from kubeflow import fairing

MODEL_FILE = 'trained_ames_model.dat'

Expand Down
File renamed without changes.
Expand Up @@ -218,7 +218,7 @@
"outputs": [],
"source": [
"import os\n",
"import fairing\n",
"from kubeflow import fairing\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
"# You can use any docker container registry istead of GCR\n",
Expand All @@ -245,11 +245,11 @@
},
"outputs": [],
"source": [
"import fairing\n",
"from fairing.builders.cluster import gcs_context\n",
"from fairing.builders.cluster.cluster import ClusterBuilder\n",
"from fairing.deployers.tfjob.tfjob import TfJob\n",
"from fairing.preprocessors.function import FunctionPreProcessor\n",
"from kubeflow import fairing\n",
"from kubeflow.fairing.builders.cluster import gcs_context\n",
"from kubeflow.fairing.builders.cluster.cluster import ClusterBuilder\n",
"from kubeflow.fairing.deployers.tfjob.tfjob import TfJob\n",
"from kubeflow.fairing.preprocessors.function import FunctionPreProcessor\n",
"\n",
"preprocessor = FunctionPreProcessor(function_obj=HousingServe, input_files=['ames_dataset/train.csv', \"requirements.txt\"])\n",
"builder = ClusterBuilder(registry=DOCKER_REGISTRY, base_image=BASE_IMAGE, preprocessor=preprocessor,\n",
Expand Down Expand Up @@ -280,12 +280,12 @@
},
"outputs": [],
"source": [
"import fairing\n",
"from fairing.builders.cluster import gcs_context\n",
"from fairing.builders.cluster.cluster import ClusterBuilder\n",
"from fairing.deployers.job.job import Job\n",
"from fairing.deployers.serving.serving import Serving\n",
"from fairing.preprocessors.function import FunctionPreProcessor\n",
"from kubeflow import fairing\n",
"from kubeflow.fairing.builders.cluster import gcs_context\n",
"from kubeflow.fairing.builders.cluster.cluster import ClusterBuilder\n",
"from kubeflow.fairing.deployers.job.job import Job\n",
"from kubeflow.fairing.deployers.serving.serving import Serving\n",
"from kubeflow.fairing.preprocessors.function import FunctionPreProcessor\n",
"\n",
"preprocessor = FunctionPreProcessor(function_obj=HousingServe, input_files=['trained_ames_model.dat', \"requirements.txt\"])\n",
"builder = ClusterBuilder(registry=DOCKER_REGISTRY, base_image=BASE_IMAGE, preprocessor=preprocessor,\n",
Expand Down
Expand Up @@ -220,7 +220,7 @@
},
"outputs": [],
"source": [
"import fairing\n",
"from kubeflow import fairing\n",
"\n",
"# Default to using the GKE backend.\n",
"FAIRING_BACKEND = 'KubeflowGKEBackend'\n",
Expand Down Expand Up @@ -252,13 +252,13 @@
" DOCKER_REGISTRY = 'gcr.io/{}/fairing-job'.format(GCP_PROJECT)\n",
" BuildContext = None\n",
"if FAIRING_BACKEND == 'KubeflowAzureBackend':\n",
" from fairing.builders.cluster.azurestorage_context import StorageContextSource\n",
" from kubeflow.fairing.builders.cluster.azurestorage_context import StorageContextSource\n",
" BuildContext = StorageContextSource(\n",
" region=AZURE_REGION, resource_group_name=AZURE_RESOURCE_GROUP,\n",
" storage_account_name=AZURE_STORAGE_ACCOUNT\n",
" )\n",
"\n",
"BackendClass = getattr(importlib.import_module('fairing.backends'), FAIRING_BACKEND)"
"BackendClass = getattr(importlib.import_module('kubeflow.fairing.backends'), FAIRING_BACKEND)"
]
},
{
Expand All @@ -278,7 +278,7 @@
},
"outputs": [],
"source": [
"from fairing import TrainJob\n",
"from kubeflow.fairing import TrainJob\n",
"train_job = TrainJob(HousingServe, input_files=['ames_dataset/train.csv', \"requirements.txt\"],\n",
" docker_registry=DOCKER_REGISTRY,\n",
" backend=BackendClass(build_context_source=BuildContext))\n",
Expand All @@ -302,7 +302,7 @@
},
"outputs": [],
"source": [
"from fairing import PredictionEndpoint\n",
"from kubeflow.fairing import PredictionEndpoint\n",
"endpoint = PredictionEndpoint(HousingServe, input_files=['trained_ames_model.dat', \"requirements.txt\"],\n",
" docker_registry=DOCKER_REGISTRY,\n",
" backend=BackendClass(build_context_source=BuildContext))\n",
Expand Down
Expand Up @@ -191,7 +191,7 @@
"outputs": [],
"source": [
"import os\n",
"import fairing\n",
"from kubeflow import fairing\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
"# You can use any docker container registry istead of GCR\n",
Expand Down
Expand Up @@ -223,8 +223,8 @@
"outputs": [],
"source": [
"import os\n",
"import fairing\n",
"from fairing import TrainJob\n",
"from kubeflow import fairing\n",
"from kubeflow.fairing import TrainJob\n",
"import sys\n",
"\n",
"# Setting up google container repositories (GCR) for storing output containers\n",
Expand Down Expand Up @@ -253,7 +253,7 @@
"train_job = TrainJob(train_and_test,\n",
" base_docker_image=BASE_IMAGE,\n",
" docker_registry=DOCKER_REGISTRY,\n",
" backend= fairing.backends.KubeflowGK(),\n",
" backend= kubeflow.fairing.backends.KubeflowGK(),\n",
" input_files=[\"requirements.txt\"])\n",
"train_job.submit()"
]
Expand All @@ -274,7 +274,7 @@
"train_job = TrainJob(lambda: train_and_test(epochs=100),\n",
" base_docker_image=BASE_IMAGE,\n",
" docker_registry=DOCKER_REGISTRY,\n",
" backend= fairing.backends.GCPManagedBackend(),\n",
" backend= kubeflow.fairing.backends.GCPManagedBackend(),\n",
" input_files=[\"requirements.txt\"])\n",
"train_job.submit()"
]
Expand Down
@@ -1,7 +1,7 @@
import os
import fairing
import tensorflow as tf

from kubeflow import fairing
# Setting up google container repositories (GCR) for storing output containers
# You can use any docker container registry istead of GCR
GCP_PROJECT = fairing.cloud.gcp.guess_project_name()
Expand Down
Expand Up @@ -6,10 +6,10 @@
"metadata": {},
"outputs": [],
"source": [
"import fairing\n",
"from fairing import TrainJob\n",
"from fairing.backends import KubeflowGKEBackend\n",
"from fairing.kubernetes.utils import get_resource_mutator"
"from kubeflow import fairing\n",
"from kubeflow.fairing import TrainJob\n",
"from kubeflow.fairing.backends import KubeflowGKEBackend\n",
"from kubeflow.fairing.kubernetes.utils import get_resource_mutator"
]
},
{
Expand Down
20 changes: 20 additions & 0 deletions kubeflow/__init__.py
@@ -0,0 +1,20 @@
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__project__ = 'kubeflow'

# TBD @jinchihe update the __version__ automatically.
__version__ = "0.0.3"

import kubeflow.fairing
8 changes: 2 additions & 6 deletions fairing/__init__.py → kubeflow/fairing/__init__.py
Expand Up @@ -2,10 +2,6 @@
from .ml_tasks.tasks import TrainJob, PredictionEndpoint

if os.getenv('FAIRING_RUNTIME', None) is not None:
from fairing.runtime_config import config
from .runtime_config import config
else:
from fairing.config import config

name = "fairing"

__version__ = "0.5.3"
from .config import config
File renamed without changes.

0 comments on commit 2fc55d2

Please sign in to comment.