Skip to content
Permalink
Browse files

add possibility to split a jenkins jobs defined with yaml in multiple…

… parts
  • Loading branch information...
sdpython committed Nov 16, 2016
1 parent 33eadff commit 49220a28a38fd097985550dc0ca12e222df61377
@@ -12,7 +12,7 @@ install:
- JENKINS_SPLIT

before_script:
- rem
- rem b

script:
- { CMD: "python -u setup.py unittests", NAME: "UT", TIMEOUT: 900 }
@@ -0,0 +1,77 @@
"""
@brief test log(time=2s)
"""

import sys
import os
import unittest
import re


try:
import src
except ImportError:
path = os.path.normpath(
os.path.abspath(
os.path.join(
os.path.split(__file__)[0],
"..",
"..")))
if path not in sys.path:
sys.path.append(path)
import src

from src.pyquickhelper.loghelper import fLOG
from src.pyquickhelper.jenkinshelper.jenkins_server import JenkinsExt
from src.pyquickhelper.jenkinshelper.jenkins_helper import setup_jenkins_server_yml

if sys.version_info[0] == 2:
FileNotFoundError = Exception


class TestYamlJenkinsSplit(unittest.TestCase):

def test_jenkins_ext_setup_server_yaml_split(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")

engines = {'Python36': 'c:\\Python36_x64',
'Python35': 'c:\\Python35_x64'}
srv = JenkinsExt(
"http://localhost:8080/", "user", "password", mock=True,
engines=engines, fLOG=fLOG, platform="win")

this = os.path.abspath(os.path.dirname(__file__))
localyml = os.path.abspath(os.path.join(
this, "data", "local.yml"))

fLOG("---------------------")
modules = [('yml', localyml, 'H H(5-6) * * 0')]
res = setup_jenkins_server_yml(srv, github="sdpython", modules=modules,
overwrite=True, add_environ=False,
location="anything")
reg = re.compile("<description>(.*)</description>")
assert len(res) > 0
for i, r in enumerate(res):
conf = r[-1]

if not conf.startswith("<?xml version='1.0' encoding='UTF-8'?>"):
raise Exception(conf)

search = reg.search(conf)
if not search:
raise Exception(conf)

job = r[0]
fLOG(search.groups()[0], "--", job, "--", r[1])

conf = conf.replace("\n", "")
exp = "</hudson.tasks.BatchFile><hudson.tasks.BatchFile>"
if exp not in conf:
raise Exception(conf)


if __name__ == "__main__":
unittest.main()
@@ -53,16 +53,24 @@ def zz_st_jconvert_sequence_into_batch_file_split(self, platform):
for r, v in res:
conv = convert_sequence_into_batch_file(
r, variables=v, platform=platform)
if not isinstance(conv, list):
raise TypeError(type(conv))
convs.append(conv)
typstr = str # unicode#
assert isinstance(conv, typstr)
assert len(res) > 0

conv = [_ for _ in convs if "SET NAME=UT" in _ and "VERSION=3.5" in _]
self.assertEqual(len(conv), 1)
conv = conv[0]
self.assertEqual(len(convs), 2)
conv = convs[0]
assert conv
fLOG(conv)
assert isinstance(conv, list)
fr = 0
for c in conv:
if "JENKINS_SPLIT" in c:
raise Exception(c)
if "pip freeze" in c:
fr += 1
fLOG("-------------------------------")
fLOG(c)
self.assertEqual(fr, 1)


if __name__ == "__main__":
@@ -452,6 +452,7 @@ def lay_build_override_newconf(t3):
# we copy javascript dependencies, reveal.js
######################################
fLOG("~~~~ JAVASCRIPT:", html_static_paths)
fLOG("~~~~ ROOT:", root_sphinxdoc)
fLOG("~~~~ BUILD:", build_paths)
for html_static_path in html_static_paths:
install_javascript_tools(
@@ -830,18 +831,19 @@ def lay_build_override_newconf(t3):
# for every layout
fLOG("[revealjs] JAVASCRIPT: COPY", html_static_paths)
fLOG("[revealjs] BUILD:", build_paths)
for html_static_path, build_path in zip(html_static_paths, build_paths):
builddoc = os.path.join(build_path, "_downloads")
if os.path.exists(builddoc):
# no download, there is probably no notebooks
# so it is not needed
fLOG("copy javascript static files from",
html_static_path, "to", builddoc)
copy = synchronize_folder(
html_static_path, builddoc, copy_1to2=True, fLOG=fLOG)
fLOG("javascript", len(copy), "files copied")
else:
fLOG("[revealjs] no need, no folder", builddoc)
for subf in ["html", "epub"]:
for html_static_path, build_path in zip(html_static_paths, build_paths):
builddoc = os.path.join(build_path, subf, "_downloads")
if os.path.exists(builddoc):
# no download, there is probably no notebooks
# so it is not needed
fLOG("copy javascript static files from",
html_static_path, "to", builddoc)
copy = synchronize_folder(
html_static_path, builddoc, copy_1to2=True, fLOG=fLOG)
fLOG("javascript", len(copy), "files copied")
else:
fLOG("[revealjs] no need, no folder", builddoc)

######
# next
@@ -24,6 +24,9 @@
from ..filehelper import read_content_ufs


_jenkins_split = "JENKINS_SPLIT"


def pickname(*l):
"""
pick the first string non null in the list
@@ -297,11 +300,14 @@ def convert_sequence_into_batch_file(seq, variables=None, platform=None):
@param seq sequence of instructions
@param variables list of variables
@param platform ``sys.platform`` if None
@return (str) batch file
@return (str) batch file or a list of batch file if the constant ``JENKINS_SPLIT``
was found in section install (this tweak is needed when the job has to be split
for Jenkins.
"""
global _jenkins_split
if platform is None:
platform = sys.platform
rows = []

iswin = platform.startswith("win")

if iswin:
@@ -314,6 +320,7 @@ def convert_sequence_into_batch_file(seq, variables=None, platform=None):
anaconda = False
conda = None
echo = "@echo" if iswin else "echo"

rowsset = []
if iswin:
rowsset.append("@echo off")
@@ -335,6 +342,9 @@ def add_path_win(rows, interpreter, pip, platform):
else:
rows.append("export PATH={0}:$PATH".format(path_pip))

rows = []
splits = [rows]

for key, value in seq:
if key == "python":
if variables.get('DIST', None) == "conda":
@@ -397,6 +407,12 @@ def add_path_win(rows, interpreter, pip, platform):
if "NAME" in value:
rows.append("set JOB_NAME=%s" % value["NAME"])
value = value["CMD"]
elif isinstance(value, list):
starter = rows.copy()
else:
raise TypeError(
"value must of type list, dict, not '{0}'".format(type(value)))

rows.append("")
rows.append(echo + " " + key.upper())
add_path_win(rows, interpreter, pip, platform)
@@ -406,19 +422,34 @@ def add_path_win(rows, interpreter, pip, platform):
keep = value
value = []
for v in keep:
value.append(v)
value.append(error_level)
if v == _jenkins_split:
rows.extend(value)
value = []
splits.append(starter.copy())
rows = splits[-1]
add_path_win(rows, interpreter, pip, platform)
else:
value.append(v)
value.append(error_level)
rows.extend(value)
elif key == 'INFO':
rowsset.append("SET {0}={1}".format(value[0], value[1]))
else:
raise ValueError("unexpected key '{0}'".format(key))
rows = rowsset + rows
try:
return "\n".join(rows)
except TypeError as e:
raise TypeError("Unexpected type\n{0}".format(
"\n".join([str((type(_), _)) for _ in rows]))) from e

splits = [rowsset + _ for _ in splits]
allres = []
for rows in splits:
try:
res = "\n".join(rows)
except TypeError as e:
raise TypeError("Unexpected type\n{0}".format(
"\n".join([str((type(_), _)) for _ in rows]))) from e
if _jenkins_split in res:
raise ValueError(
"Constant '{0}' is present in the generated script. It can only be added to the install section.".format(_jenkins_split))
allres.append(res)
return allres if len(allres) > 1 else allres[0]


def infer_project_name(file_or_buffer, source):
@@ -493,7 +524,10 @@ def enumerate_processed_yml(file_or_buffer, context=None, engine="jinja2", platf
if server is not None:
name = "_".join([project_name, var.get('NAME', ''), typstr(var.get("VERSION", '')).replace(".", ""),
var.get('DIST', '')])
conv = "SET NAME_JENKINS=" + name + "\n" + conv
if isinstance(conv, list):
conv = ["SET NAME_JENKINS=" + name + "\n" + _ for _ in conv]
else:
conv = "SET NAME_JENKINS=" + name + "\n" + conv
timeout = var.get("TIMEOUT", None)
import jenkins
try:

0 comments on commit 49220a2

Please sign in to comment.
You can’t perform that action at this time.