support CMDPY and others options

_doc/examples/set_jenkins_jobs.py

@@ -0,0 +1,82 @@
+"""
+================
+Set Jenkins Jobs
+================
+
+This jobs is used to add jobs to a Jenkins server in order
+to build packages on Windows. The YAML definition is located
+at `pymyinstall/whl <https://github.com/sdpython/pymyinstall/tree/master/whl>`_
+and it defines the jobs in this folder
+`pymyinstall/whl/windows <https://github.com/sdpython/pymyinstall/tree/master/whl/windows>`_.
+
+"""
+################################################
+# imports
+import os
+try:
+    import pyquickhelper
+except ImportError:
+    import sys
+    sys.path.append("src")
+    import pyquickhelper
+
+from pyquickhelper.jenkinshelper import setup_jenkins_server_yml, JenkinsExt
+from pyquickhelper.loghelper import fLOG
+
+#################################
+# Starts logging.
+fLOG(OutputPrint=True)
+fLOG("start")
+
+#################################
+# password
+import keyring
+user = keyring.get_password("jenkins", os.environ["COMPUTERNAME"] + "user")
+pwd = keyring.get_password("jenkins", os.environ["COMPUTERNAME"] + "pwd")
+
+#################################
+# local path
+engines = dict(Python35="c:\\Python35_x64",
+               Python36="c:\\Python36_x64")
+folder = "C:\\%s\github\\pymyinstall\\whl" % os.environ["USERNAME"]
+location = "d:\\jenkins\\pymy"
+
+#################################
+if not os.path.exists("build"):
+    os.mkdir("build")
+
+#################################
+# Loads the yml template.
+yml = os.path.join(folder, ".local.jenkins.win.yml")
+with open(yml, "r", encoding="utf-8") as f:
+    content = f.read()
+
+#################################
+# Starts the Jenkins server.
+js = JenkinsExt('http://localhost:8080/', user,
+                pwd, fLOG=fLOG, engines=engines)
+
+#################################
+# Defiens the Jenkins job.
+ymls = []
+for mod in ["polylearn", "dynd-python"]:
+    new_content = content.replace("__MODULE__", mod)
+    yml = os.path.join("build", ".local.jenkins.win.{0}.yml".format(mod))
+    with open(yml, "w", encoding="utf-8") as f:
+        f.write(new_content)
+    batch = os.path.join(folder, "windows", "build_{0}.bat".format(mod))
+    with open(batch, "r", encoding="utf-8") as f:
+        cbat = f.read()
+    with open(yml, "w", encoding="utf-8") as f:
+        f.write(new_content)
+    toadd = [dict(name="build_{0}.bat".format(mod),
+                  content=cbat)]
+    ymls.append(("yml", yml, dict(scripts=toadd)))
+
+#################################
+# Update the Jenkins jobs for the given set of modules
+fLOG("Update jenkins")
+setup_jenkins_server_yml(js, github=None, modules=ymls,
+                         overwrite=True, location=location, prefix="",
+                         delete_first=False, disable_schedule=False, fLOG=fLOG)
+fLOG("Done")

_unittests/ut_jenkinshelper/test_yaml.py

@@ -113,7 +113,7 @@ def test_jenkins_job_multiplication(self):
         doc = [[s[0] for s in seq if s[1] is not None] for seq, _ in res]
         fLOG(doc)
         doc = [s for s in doc if "documentation" in s]
-        if len(doc) != 3:
+        if len(doc) != 2:
             raise Exception("\n".join(str(_) for _ in doc))
         else:
             fLOG("**", doc)
@@ -352,13 +352,6 @@ def test_jconvert_sequence_into_batch_file27(self):
             if %errorlevel% neq 0 exit /b %errorlevel%
             cd ..
             if %errorlevel% neq 0 exit /b %errorlevel%
-
-            @echo DOCUMENTATION
-            set PATH=ROOT\\%NAME_JENKINS%\\_venv\\Scripts;%PATH%
-            python -u setup.py build_sphinx
-            if %errorlevel% neq 0 exit /b %errorlevel%
-            xcopy /E /C /I /Y _doc\\sphinxdoc\\build\\html dist\\html
-            if %errorlevel% neq 0 exit /b %errorlevel%
             """.replace("            ", "").strip("\n \t\r")
             val = conv.strip("\n \t\r")
             if expected != val:

src/pyquickhelper/jenkinshelper/jenkins_server.py

@@ -32,7 +32,7 @@
 from ..pycode.windows_scripts import windows_jenkins_27_conda, windows_jenkins_27_def
 from ..pycode.build_helper import private_script_replacements
 from .jenkins_exceptions import JenkinsExtException, JenkinsJobException
-from .jenkins_server_template import _config_job, _trigger_up, _trigger_time, _git_repo, _task_batch, _publishers
+from .jenkins_server_template import _config_job, _trigger_up, _trigger_time, _git_repo, _task_batch, _publishers, _file_creation
 from .yaml_helper import enumerate_processed_yml
 
 _timeout_default = 1200
@@ -282,7 +282,7 @@ def get_engine_from_job(self, job, return_key=False):
             res = self.engines["default"]
             key = "default"
         if res is None:
-            raise JenkinsJobException("unable to find engine in job {}, available: {}".format(
+            raise JenkinsJobException("Unable to find engine in job '{}', available: {}".format(
                 job, ", ".join(self.engines.keys())))
         else:
             if "[27]" in job and "python34" in res.lower():
@@ -659,6 +659,9 @@ def create_job_template(self, name, git_repo, credentials="", upstreams=None, sc
 
         .. versionchanged:: 1.4
             Parameter *adjust_scheduler* was added to delayed some jobs if a spot is already taken.
+
+        .. versionchanged:: 1.5
+            *kwargs* can contain option *job_options*.
         """
         if 'platform' in kwargs:
             raise NameError(
@@ -701,7 +704,7 @@ def create_job_template(self, name, git_repo, credentials="", upstreams=None, sc
         else:
             trigger = ""
 
-        # cmd = "set" if self.platform.startswith("win") else "export"
+        job_options = kwargs.get('job_options', None)
 
         if not isinstance(script, list):
             script = [script]
@@ -726,8 +729,25 @@ def create_job_template(self, name, git_repo, credentials="", upstreams=None, sc
                 .replace("__GITREPO__", git_repo) \
                 .replace("__CRED__", "<credentialsId>%s</credentialsId>" % credentials)
 
+        # additional scripts
+        before = []
+        if job_options is not None:
+            if 'scripts' in job_options:
+                lscripts = job_options['scripts']
+                for scr in lscripts:
+                    au = _file_creation.replace("__FILENAME__", scr["name"]) \
+                                       .replace("__CONTENT__", scr["content"])
+                    if "__" in au:
+                        raise Exception(
+                            "Unable to fully replace expected string in:\n{0}".format(au))
+                    before.append(au)
+                del job_options['scripts']
+            if len(job_options) > 0:
+                raise ValueError(
+                    "Unable to process options\n{0}".format(job_options))
+
         # scripts
-        tasks = [JenkinsExt._task_batch.replace(
+        tasks = before + [JenkinsExt._task_batch.replace(
             "__SCRIPT__", s) for s in script_mod]
 
         # location
@@ -998,6 +1018,13 @@ def setup_jenkins_server(self, github, modules, get_jenkins_script=None, overwri
         indexes = dict(order=0, dozen="A")
         counts = {}
         for jobs in modules:
+            if isinstance(jobs, tuple):
+                if len(jobs) == 0:
+                    raise ValueError("Empty jobs in the list.")
+                if jobs[0] == "yml" and len(jobs) != 3:
+                    raise ValueError(
+                        "If it is a yml jobs, the tuple should contain 3 elements: ('yml', filename, schedule or None or dictionary).\nNot: {0}".format(jobs))
+
             cre, ds, locs = self._setup_jenkins_server_modules_loop(jobs=jobs, counts=counts,
                                                                     get_jenkins_script=get_jenkins_script,
                                                                     location=location, adjust_scheduler=adjust_scheduler,
@@ -1067,7 +1094,12 @@ def _setup_jenkins_server_job_iteration(self, job, get_jenkins_script, location,
                 options = {}
 
             # job and scheduler
-            job, scheduler = job[:2]
+            job, scheduler_options = job[:2]
+            if isinstance(scheduler_options, dict):
+                scheduler = scheduler_options.get('scheduler', None)
+            else:
+                scheduler = scheduler_options
+                scheduler_options = None
             if scheduler is not None:
                 order = 1
                 if counts.get(dozen, 0) > 0:
@@ -1219,7 +1251,8 @@ def _setup_jenkins_server_job_iteration(self, job, get_jenkins_script, location,
                                                          add_environ=add_environ, server=self, git_repo=gitrepo, scheduler=scheduler,
                                                          description=description, credentials=cred, success_only=success_only,
                                                          timeout=timeout, platform=self.platform, adjust_scheduler=adjust_scheduler,
-                                                         overwrite=overwrite, build_location=location, mails=self.mails):
+                                                         overwrite=overwrite, build_location=location, mails=self.mails,
+                                                         job_options=scheduler_options):
                 if name in done:
                     s = "A name '{0}' was already used for a job, from:\n{1}\nPROCESS:\n{2}"
                     raise ValueError(

src/pyquickhelper/jenkinshelper/jenkins_server_template.py

@@ -102,11 +102,23 @@
 </hudson.tasks.BatchFile>
 """
 
-# mails
+#: mails
 _publishers = """
 <hudson.tasks.Mailer plugin="mailer">
     <recipients>__MAIL__</recipients>
     <dontNotifyEveryUnstableBuild>false</dontNotifyEveryUnstableBuild>
     <sendToIndividuals>true</sendToIndividuals>
 </hudson.tasks.Mailer>
 """
+
+#: creation of a file
+_file_creation = """
+<com.etas.jenkins.plugins.CreateTextFile.CreateFileBuilder plugin="text-file-operations">
+  <textFilePath>__FILENAME__</textFilePath>
+  <textFileContent>
+__CONTENT__
+  </textFileContent>
+  <fileOption>overWrite</fileOption>
+  <useWorkspace>true</useWorkspace>
+</com.etas.jenkins.plugins.CreateTextFile.CreateFileBuilder>
+"""

src/pyquickhelper/jenkinshelper/yaml_helper.py

@@ -107,7 +107,11 @@ def evaluate_condition(cond, variables=None):
         e = eval(cond)
         return all(e)
     else:
-        return eval(cond)
+        try:
+            return eval(cond)
+        except SyntaxError as e:
+            raise SyntaxError(
+                "Unable to interpret '{0}'\nvariables: {1}".format(cond, variables)) from e
 
 
 def interpret_instruction(inst, variables=None):
@@ -120,7 +124,7 @@ def interpret_instruction(inst, variables=None):
 
     Example of a statement::
 
-        if [ ${PYTHON} == "C:\\\\Python36_x64" ]; then python setup.py build_sphinx; fi
+        if [ ${PYTHON} == "C:\\\\Python36_x64" ] then python setup.py build_sphinx fi
     """
     if isinstance(inst, list):
         res = [interpret_instruction(_, variables) for _ in inst]
@@ -239,7 +243,7 @@ def enumerate_convert_yaml_into_instructions(obj, variables=None, add_environ=Tr
                 value = interpret_instruction(value[i_script], variables)
                 if isinstance(value, dict):
                     for k, v in sorted(value.items()):
-                        if k != "CMD":
+                        if k not in ('CMD', 'CMDPY'):
                             seq.append(('INFO', (k, v)))
                             variables[k] = v
 
@@ -348,6 +352,7 @@ def add_path_win(rows, interpreter, pip, platform):
 
     for key, value in seq:
         if key == "python":
+            variables["YMLPYTHON"] = value
             if variables.get('DIST', None) == "conda":
                 rows.append(echo + " conda")
                 anaconda = True
@@ -402,12 +407,16 @@ def add_path_win(rows, interpreter, pip, platform):
         elif key in {"install", "before_script", "script", "after_script", "documentation"}:
             if value is not None:
                 if isinstance(value, dict):
-                    if "CMD" not in value:
+                    if "CMD" not in value and "CMDPY" not in value:
                         raise KeyError(
-                            "A script defined by a dictionary must contain key '{0}' in \n{1}".format("CMD", value))
+                            "A script defined by a dictionary must contain key '{0}' or '{1}' in \n{2}".format("CMD", 'CMDPY', value))
                     if "NAME" in value:
                         rows.append("set JOB_NAME=%s" % value["NAME"])
-                    value = value["CMD"]
+                    if "CMD" in value:
+                        value = value["CMD"]
+                    else:
+                        value = evaluate_condition(
+                            value["CMDPY"], variables=variables)
                 elif isinstance(value, list):
                     starter = list(rows)
                 elif isinstance(value, typstr):
@@ -537,7 +546,8 @@ def enumerate_processed_yml(file_or_buffer, context=None, engine="jinja2", platf
 
         # we extract a suffix from the command line
         if server is not None:
-            name = "_".join([project_name, var.get('NAME', ''), typstr(var.get("VERSION", '')).replace(".", ""),
+            name = "_".join([project_name, var.get('NAME', ''),
+                             typstr(var.get("VERSION", '')).replace(".", ""),
                              var.get('DIST', '')])
             if isinstance(conv, list):
                 conv = ["SET NAME_JENKINS=" + name + "\n" + _ for _ in conv]
@@ -563,7 +573,7 @@ def enumerate_processed_yml(file_or_buffer, context=None, engine="jinja2", platf
                         fLOG("[jenkins] delete job", name)
                     server.delete_job(name)
 
-            if project_name not in git_repo:
+            if git_repo is not None and project_name not in git_repo:
                 git_repo += project_name
 
             # set up location