Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 407 lines (298 sloc) 12.039 kb
28f12ae Added Ghetto CI
Mikko authored
1 #! /usr/bin/python3
2 """
3
4 Continuous integration server, ghetto style
5
6 What it is
7 --------------
8
9 Python script in few hundred lines fullfilling your dirty little continuos integration needs.
10
11 What it does
12 --------------
13
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
14 1. Runs update on a source folder from a Subversion repository (note: VCS backend easy to change)
28f12ae Added Ghetto CI
Mikko authored
15
16 2. See if there are new commits since the last run
17
18 3. Run tests if the source code in fact was changed
19
20 4. See if the tests status has changed from success to failure or vice versa
21
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
22 5. Send email notifications to the team that now shit has hit the fan
28f12ae Added Ghetto CI
Mikko authored
23
24 Why to use
25 ---------------
26
27 To improve your software project quality and cost effectiveness,
28 you want to automatically detect bad commits breaking your project
29 `without need to install 48 MB of Java software <http://jenkins-ci.org/>`_.
30 This script is mostly self-contained, easy to understand, easy to hack into pieces and customize for your very own need.
31
32 How to use
33 --------------
34
35 Create Python 3 virtualenv and run ghetto script using Python interpreter configured under this virtualenv::
36
37 # We install directly under our UNIX user home folder
38 cd ~
39 virtualenv -p python3.2 ghetto
40 source ghetto/bin/activate
41 pip install plac
42
43 # Install ghetto-ci.py command by downloading it from Github using wget
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
44 wget --no-check-certificate -O ghetto-ci.py https://raw.github.com/miohtama/vvv/master/ghettoci/main.py
28f12ae Added Ghetto CI
Mikko authored
45
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
46 # Running the script, using the Python environment prepared
47 # to see that everything works
28f12ae Added Ghetto CI
Mikko authored
48 ghetto/bin/python ghettoci.py
49
50 Now you need have
51
52 * A software repository folder. This must be pre-checked out Subversion repository where
53 you can run ``svn up`` command.
54
55 * A command to execute unit tests and such. Must return process exit code 0 on success.
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
56 If you don't bother write tests, at least `lint and validate your source code <http://pypi.python.org/pypi/vvv>`_.
28f12ae Added Ghetto CI
Mikko authored
57
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
58 * A file storing the test status. Status file keeps track whether the last round
28f12ae Added Ghetto CI
Mikko authored
59 or tests succeeded or failed. You'll get email reports only when the test status changed -
60 there is little need to get "tests succeeded" email for every commit.
61
62 * Email server details to send out notifications. Gmail works perfectly.
63
64 Example of a command::
65
66 # Will print output to console because email notification details are not given
67 ghetto/bin/python ghettoci.py
68
69 Then just make *ghettoci* to poll the repository in UNIX crom (Ubuntu example).
70 Create file ``/etc/cron.hourly/ghetto-ci``::
71
72
73 ... or just use OSX or Windows task automators.
74
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
75 Source and credits
28f12ae Added Ghetto CI
Mikko authored
76 --------------------
77
78 `Ghetto CI lives on Github, in VVV project <https://github.com/miohtama/vvv>`_.
79
80 Mikko Ohtamaa, the original author (`blog <http://opensourcehacker.com>`_, `Twitter <http://twitter.com/moo9000>`_)
81
82 Licensed under `WTFPL <http://sam.zoy.org/wtfpl/COPYING>`_.
83
84 `Ghetto style explained <http://www.urbandictionary.com/define.php?term=ghetto+style>`_.
85
86 """
87
88 # pylint complains about abstract Python 3 members and ABCMeta, this will be fixed in the future
89 # pylint: disable=R0201, W0611
90
91 # Python imports
92 from abc import ABCMeta, abstractmethod
93 from email.mime.text import MIMEText
94 import pickle
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
95 import os
28f12ae Added Ghetto CI
Mikko authored
96 import sys
97 import subprocess
98 from smtplib import SMTP_SSL, SMTP
99
100 def shell(cmdline):
101 """
102 Execute a shell command.
103
104 :returns: (exitcode, stdout / stderr output as string) tuple
105 """
106
107 process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
108
109 # XXX: Support stderr interleaving
110 out, err = process.communicate()
111
112 # :E1103: *%s %r has no %r member (but some types could not be inferred)*
113 # pylint: disable=E1103
114 out = out.decode("utf-8")
115 err = err.decode("utf-8")
116
117 return (process.returncode, out + err)
118
119 class Status:
120 """
121 Store CI status of a test run.
122
123 Use Python pickling serialization for making status info persistent.
124 """
125
126 def __init__(self):
127 self.test_success = False
128 self.last_commit_id = None
129
130 @classmethod
131 def read(cls, path):
132 """
133 Read status file.
134
135 Return fresh status if file does not exist.
136 """
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
137
138 if not os.path.exists(path):
28f12ae Added Ghetto CI
Mikko authored
139 # Status file do not exist, get default status
140 return Status()
141
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
142 f = open(path, "rb")
143
28f12ae Added Ghetto CI
Mikko authored
144 try:
145 content = f.read()
146 return pickle.loads(content)
147 finally:
148 f.close()
149
150 @classmethod
151 def write(cls, path, status):
152 """
153 Write status file
154 """
155 f = open(path, "wt")
156 content = pickle.dumps(status)
157 f.write(content)
158 f.close()
159
160
161 class Repo(metaclass=ABCMeta):
162 """
163 Define interface for presenting one monitored software repository in ghetto-ci.
164 """
165 def __init__(self, path):
166 """
167 :param path: Abs FS path to monitored repository
168 """
169 self.path = path
170
171 @abstractmethod
172 def update(self):
173 """ Update repo from version control """
174 pass
175
176 @abstractmethod
177 def get_last_commit_info(self):
178 """
179 Get the last commit status.
180
181 :return tuple(commit id, commiter, message, raw_output) or (None, None, None, raw_output) on error
182 """
183 return (None, None, None)
184
185 class SVNRepo(Repo):
186 """ Handle Subversion repository update and last commit info extraction """
187
188 def update(self):
189 """
190 Run repo update in a source folder
191 """
192 exitcode, output = shell("svn up %s" % self.path)
193 return exitcode == 0, output
194
195 def get_last_commit_info(self):
196 """
197 Get the last commit status.
198 """
199 info, output = self.get_svn_info()
200 if not info:
201 return (None, None, None, output)
202
203 return info["Revision"]
204
205 def get_svn_info(self):
206 """
207 Get svn info output parsed to dict
208 """
209 exit_code, output = shell("svn info %s" % self.path)
210 if exit_code != 0:
211 return None, output
212
213 data = {}
214 for line in output.split("\n"):
215 key, values = line.split(":")
216 value = ",".join(values)
217 data[key] = value
218
219 return data, output
220
221 class Notifier:
222 """
223 Intelligent spam being. Print messages to stdout and send emai if
224 SMTP server details are available.
225 """
226
227 def __init__(self, server, port, username, password, receivers, from_address):
228 """
229 :param server: SMTP server
230
231 :param port: SMTP port, autodetects SSL
232
233 :param username: SMTP credentials
234
235 :param password: SMTP password
236
237 :param receivers: String, comma separated list of receivers
238
239 :param from_email: Sender's email address
240 """
241
242 self.server = server
243 self.port = port
244 self.username = username
245 self.password = password
246 self.receivers = receivers
247 self.from_address = from_address
248
249 # This would be nice trick, but let's keep linter happy
250 # self.__dict__.update(kwargs)
251
252 def send_email_notification(self, subject, output):
253 """
254
255 Further info:
256
257 * http://docs.python.org/py3k/library/email-examples.html
258
259 :param receivers: list of email addresses
260
261 :param subject: Email subject
262
263 :param output: Email payload
264 """
265
266 if self.port == 465:
267 # SSL encryption from the start
268 smtp = SMTP_SSL(self.server, self.port)
269 else:
270 # Plain-text SMTP, or opt-in to SSL using starttls() command
271 smtp = SMTP(self.server, self.port)
272
273 msg = MIMEText(output, "text/plain")
274 msg['Subject'] = subject
275 msg['From'] = self.from_address
276
277 # SMTP never works on the first attempt...
278 smtp.set_debuglevel(True)
279
280 # SMTP authentication is optional
281 if self.username and self.password:
282 smtp.login(self.username, self.password)
283
284 try:
285 smtp.sendmail(self.from_address, self.receivers, msg.as_string())
286 finally:
287 smtp.close()
288
289 def print_notification(self, subject, output):
290 """
291 Dump the notification to stdout
292 """
293 print(subject)
294 print("-" * len(subject))
295 print(output)
296
297 def notify(self, subject, output):
298 """
299 Notify about the tests status.
300 """
301 if self.server:
302 self.send_email_notification(subject, output)
303
304 self.print_notification(subject, output)
305
306 def run_tests(test_command):
307 """
308 Run testing command.
309
310 Assume exit code = 0 -> test success
311 """
312 exitcode, output = shell(test_command)
313 return (exitcode == 0, output)
314
315
316 # Parsing command line with plac rocks
317 # http://plac.googlecode.com/hg/doc/plac.html
318 def main(smtpserver : ("SMTP server address for mail out", "option"),
319 smtpport : ("SMTP server port for mail out", "option", None, int),
320 smtpuser : ("SMTP server username", "option"),
321 smtppassword : ("SMTP server password", "option"),
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
322 smtpfrom : ("Notification email From address", "option"),
323 receivers : ("Notification email receives as comma separated string", "option"),
28f12ae Added Ghetto CI
Mikko authored
324 force : ("Run tests regardless if there have been any repository updates", "flag"),
325
326 repository : ("Monitored source control repository (SVN)", "positional"),
327 statusfile : ("Status file to hold CI history of tests", "positional"),
328 testcommand : ("Command to run tests. Exit code 0 indicates test success", "positional"),
329 ):
330 """
331 ghetto-ci
332
333 A simple continous integration server.
334
335 ghetto-ci will monitor the software repository.
336 Give a (Subversion) software repository and a test command run test against it.
337 Make this command run regularly e.g. using UNIX cron service.
338 You will get email notification when test command status changes from exit code 0.
339
340 For more information see https://github.com/miohtama/vvv
341 """
342
343
344 notifier = Notifier(server=smtpserver, port=smtpport,
732fa38 Use local Sphinx command always. Documentation updates
Mikko authored
345 username=smtpuser, password=smtppassword, from_address=smtpfrom,
28f12ae Added Ghetto CI
Mikko authored
346 receivers=receivers)
347
348 repo = SVNRepo(repository)
349 status = Status.read(statusfile)
350
351 success, output = repo.update()
352
353 # Handle repo update failure
354 if not success:
355 notifier.notify("Could not update repository: %s. Probably not valid SVN repo?\n" % repository, output)
356 return 1
357
358 commit_id, commit_author, commit_message, output = repo.get_last_commit_info()
359
360 # Handle repo info failure
361 if not commit_id:
362 notifier.notify("Could not get commit info: %s\n%s" % (repository, output), "Check svn info by hand")
363 return 1
364
365 # See if repository status has changed
366 if commit_id != status.last_commit_id or force:
367 test_success, output = run_tests(testcommand)
368 else:
369 # No new commits, nothing to do
370 return 0
371
372 # Decorate test run output with commit info as the header
373 output = "Last commit %s: %s by %s\n %s" % (commit_id, commit_message, commit_author, output)
374
375 # Test run status have changed since last run
376 if test_success != status.test_success:
377
378 if test_success:
379 subject = "Test now succeed @ %s" % repository
380 else:
381 subject = "Test now fail @ %s" % repository
382
383 notifier.notify(subject, output)
384
385 # Update persistent test status
386 new_status = Status()
387 new_status.last_commit_id = commit_id
388 new_status.test_success = test_success
389 Status.write(statusfile, new_status)
390
391 if test_success:
392 return 0
393 else:
394 return 1
395
396 def entry_point():
397 """
398 Enter the via setup.py entry_point declaration.
399
400 Handle UNIX style application exit values
401 """
402 import plac
403 exitcode = plac.call(main)
404 sys.exit(exitcode)
405
406 if __name__ == "__main__":
407 entry_point()
Something went wrong with that request. Please try again.