This repository has been archived by the owner on Jan 13, 2024. It is now read-only.
/
jenkins_helper.py
185 lines (164 loc) · 7.69 KB
/
jenkins_helper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
"""
@file
@brief Set up a jenkins server with all the necessary job
"""
import os
import re
import sys
from pyquickhelper.loghelper import noLOG
from pyquickhelper.jenkinshelper import setup_jenkins_server_yml
from pyquickhelper.jenkinshelper.yaml_helper import load_yaml
from .teaching_modules import get_teaching_modules
def get_interpreter(platform=None):
"""
Returns the default interpreter.
@param platform platform
"""
if platform == sys.platform:
return os.path.dirname(sys.executable)
elif platform.startswith("win"):
return "c:\\Python%d%d%d_x64" % sys.version_info[:3]
elif platform.startswith("darwin"):
return "/usr/local/bin"
return "/usr/local/bin"
def engines_default(prefix="c:\\", prefix_python="c:\\", prefix_conda="c:\\", platform=None):
"""
Returns a dictionary with default values for a :epkg:`Jenkins` server.
@param prefix prefix for Jenkins location
@param prefix_python prefix for Python distribution
@param prefix_conda prefix for Anaconda or Miniconda distribution
@param platform platform to use
@return dictionary
.. warning::
Virtual environment with conda must be created on the same disk
as the original interpreter. The other scenario is not supported.
"""
if platform is None:
platform = sys.platform
if platform.startswith("win"):
res = dict(anaconda2=os.path.join(prefix_conda, "Anaconda2"),
anaconda3=os.path.join(prefix_conda, "Anaconda3"),
default=get_interpreter(platform=platform),
py38=get_interpreter(platform=platform),
py37=get_interpreter(platform=platform),
py36=os.path.join(prefix_python, "Python36_x64"),
py27=os.path.join(prefix_python, "Python27_x64"),
Python38pyq=os.path.join(
prefix, "jenkins", "venv", "py38", "pyq", "Scripts"),
Python37pyq=os.path.join(
prefix, "jenkins", "venv", "py37", "pyq", "Scripts"),
Python36pyq=os.path.join(prefix, "jenkins", "venv", "py36", "pyq", "Scripts"))
res["Python27"] = res["py27"]
res["Python36"] = res["py36"]
res["Python37"] = res["py37"]
res["Python38"] = res["py38"]
res["Anaconda2"] = res["anaconda2"]
res["Anaconda3"] = res["anaconda3"]
return res
res = {}
for k in [-1, 0, 1]:
vers = (sys.version_info[0], sys.version_info[1] + k)
key = "Python%d%d" % vers
res[key] = get_interpreter(platform=platform)
res["py%d%d" % vers] = res[key]
return res
def default_jenkins_jobs(filter=None, neg_filter=None, root=None, platform=None):
"""
Default list of :epkg:`Jenkins` jobs.
@param filter keep a subset of jobs (regular expression)
@param neg_filter remove a subset of jobs (regular expression)
@param root where to find yml project
@param platform platform or None for the current one
@return list
It produces a subset of the following list of jobs:
.. runpython::
from ensae_teaching_cs.automation.jenkins_helper import default_jenkins_jobs
modules = default_jenkins_jobs()
text = [str(m) for m in modules]
print("\\n".join(text))
"""
if platform is None:
platform = sys.platform
plat = "win" if platform.startswith("win") else "lin"
yml = []
pattern = "https://raw.githubusercontent.com/sdpython/%s/master/.local.jenkins.{0}.yml".format(
plat)
modules = ["_automation"] + get_teaching_modules()
for c in modules:
yml.append(pattern % c)
if filter is not None or neg_filter is not None:
reg = re.compile(filter if filter else ".*")
neg_reg = re.compile(neg_filter if neg_filter else "^$")
res = default_jenkins_jobs(platform=platform)
new_res = []
for row in res:
if isinstance(row, str):
if reg.search(row) and not neg_reg.search(row):
new_res.append(row)
elif isinstance(row, tuple):
if reg.search(row[0]) and not neg_reg.search(row[0]):
new_res.append(row)
elif isinstance(row, list):
# list
sub = []
for item in row:
if isinstance(item, str):
if reg.search(item) and not neg_reg.search(item):
sub.append(item)
elif isinstance(item, tuple):
if reg.search(item[0]) and not neg_reg.search(item[0]):
sub.append(item)
else:
raise TypeError("{0} - {1}".format(item, type(item)))
if len(sub) > 0:
new_res.append(sub)
else:
raise TypeError("{0} - {1}".format(row, type(row)))
return new_res
else:
context = {'Python37': 'python3.7',
'Python38': 'python3.8', 'Python39': 'python3.9'}
yml_data = load_yaml(pattern % '_automation', context=context)
pyth = yml_data[0]['python']
res = []
for pyt in pyth:
v = pyt['VERSION'] + 0.01
vers = (int(v), int((v - int(v)) * 10))
res.extend(["standalone [local_pypi] [py%d%d]" % vers,
("pymyinstall [update_modules] [py%d%d]" % vers,
"H H(0-1) * * 5")])
res.extend(('yml', c, 'H H(0-1) * * %d' % (i % 7))
for i, c in enumerate(yml))
return res
def setup_jenkins_server(js, github="sdpython", modules=None,
overwrite=False, location=None, prefix="",
delete_first=False, disable_schedule=False,
fLOG=noLOG):
"""
Sets up many jobs on :epkg:`Jenkins`.
@param js (JenkinsExt) jenkins server (specially if you need credentials)
@param github github account if it does not start with *http://*,
the link to git repository of the project otherwise
@param modules modules for which to generate the :epkg:`Jenkins` job
(see @see fn default_jenkins_jobs which provides the default value
if *modules* is None)
@param overwrite do not create the job if it already exists
@param location None for default or a local folder
@param prefix add a prefix to the name
@param delete_first remove all jobs first
@param disable_schedule disable schedule for all jobs
@param fLOG logging function
@return list of created jobs
*modules* is a list defined as follows:
* each element can be a string or a tuple (string, schedule time) or a list
* if it is a list, it contains a list of elements defined as previously
* the job at position i is not scheduled, it will start after the last
job at position i-1 whether or not it fails
"""
platform = js.platform
if modules is None:
modules = default_jenkins_jobs(platform=platform)
r = setup_jenkins_server_yml(js, github=github, modules=modules, get_jenkins_script=None,
overwrite=overwrite, location=location, prefix=prefix,
delete_first=delete_first, disable_schedule=disable_schedule)
return r