/
setup.py
120 lines (97 loc) · 3.87 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!/usr/bin/env python
""" Amazon SageMaker Debugger is an offering from AWS which helps you automate the debugging of machine learning training jobs.
This library powers Amazon SageMaker Debugger, and helps you develop better, faster and cheaper models by catching common errors quickly.
It allows you to save tensors from training jobs and makes these tensors available for analysis, all through a flexible and powerful API.
It supports TensorFlow, PyTorch, MXNet, and XGBoost on Python 3.6+.
- Zero Script Change experience on SageMaker when using supported versions of SageMaker Framework containers or AWS Deep Learning containers
- Full visibility into any tensor which is part of the training process
- Real-time training job monitoring through Rules
- Automated anomaly detection and state assertions
- Interactive exploration of saved tensors
- Distributed training support
- TensorBoard support
"""
# Standard Library
import os
import sys
from datetime import date
# Third Party
import setuptools
exec(open("smdebug/_version.py").read())
CURRENT_VERSION = __version__
DOCLINES = (__doc__ or "").split("\n")
FRAMEWORKS = ["tensorflow", "pytorch", "mxnet", "xgboost"]
TESTS_PACKAGES = ["pytest", "torchvision", "pandas"]
INSTALL_REQUIRES = [
"protobuf>=3.6.0",
"numpy>=1.16.0",
"packaging",
"boto3>=1.10.32",
"pyinstrument>=3.1.3",
]
def compile_summary_protobuf():
proto_paths = ["smdebug/core/tfevent/proto"]
cmd = "set -ex && protoc "
for proto_path in proto_paths:
proto_files = os.path.join(proto_path, "*.proto")
cmd += proto_files + " "
print("compiling protobuf files in {}".format(proto_path))
cmd += " --python_out=."
return os.system(cmd)
def build_package(version):
packages = setuptools.find_packages(include=["smdebug", "smdebug.*"])
setuptools.setup(
name="smdebug",
version=version,
long_description="\n".join(DOCLINES[1:]),
long_description_content_type="text/markdown",
author="AWS DeepLearning Team",
description=DOCLINES[0],
url="https://github.com/awslabs/sagemaker-debugger",
packages=packages,
classifiers=[
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3 :: Only",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
],
install_requires=INSTALL_REQUIRES,
setup_requires=["pytest-runner"],
tests_require=TESTS_PACKAGES,
python_requires=">=3.6",
license="Apache License Version 2.0",
)
if compile_summary_protobuf() != 0:
print(
"ERROR: Compiling summary protocol buffers failed. You will not be able to use smdebug. "
"Please make sure that you have installed protobuf3 compiler and runtime correctly."
)
sys.exit(1)
def scan_git_secrets():
import subprocess
import os
import shutil
def git(*args):
return subprocess.call(["git"] + list(args))
shutil.rmtree("/tmp/git-secrets", ignore_errors=True)
git("clone", "https://github.com/awslabs/git-secrets.git", "/tmp/git-secrets")
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir("/tmp/git-secrets")
subprocess.check_call(["make"] + ["install"])
os.chdir(dir_path)
git("secrets", "--install")
git("secrets", "--register-aws")
return git("secrets", "--scan", "-r")
if scan_git_secrets() != 0:
import sys
sys.exit(1)
def detect_smdebug_version():
if "--release" in sys.argv:
sys.argv.remove("--release")
return CURRENT_VERSION
return CURRENT_VERSION + "b" + str(date.today()).replace("-", "")
version = detect_smdebug_version()
build_package(version=version)