Save progress

Dockerfile

@@ -1,10 +1,8 @@
-FROM python:3-slim
+FROM python:3
 
 ADD . /code
 WORKDIR /code
 
-RUN apt-get update -qq
-RUN apt-get install gcc -y
 RUN pip install -r requirements.txt
 
 ENV PYTHONUNBUFFERED=1

Makefile

@@ -0,0 +1,11 @@
+clean:
+	bash misc/cleanup_jobs_dir.sh
+
+tests: clean
+	nosetests
+
+build: clean
+	docker build -t agrrh/pieter-ci:dev .
+
+publish: clean build
+	docker push agrrh/pieter-ci:dev

README.md

@@ -1,17 +1,77 @@
+# Rationale
+
+This is Pieter, a bear (yes, a bear) and a super-lightweight CI.
+
+One day I realized that number of great tools doesn't suit one simple idea: **to provide lightweight private CI**
+
+- [TeamCity](https://jetbrains.ru/products/teamcity/) and [Jenkins](https://jenkins-ci.org/) are way too heavy
+- [Travis](https://travis-ci.org/) not suitable when one needs privacy
+- [Drone](https://drone.io/) is container-oriented so sometimes an overhead
+
+There are many other solutions, but it was hard to find one good enough to rapidly deploy and use in a minute or two. So meet the Pieter CI!
+
+*Pieter pronounces more like St. Petersburg in short form, not like a person's name.*
+
+# Key concepts
+
+- **Lightweight**, daemon would never consume more than couple percents of your CPU
+- **Compact**, tool designed to solve one and only issue: executing predefined jobs on demand
+- **Easy**, absence of complex stuff and overmanagement makes deploy, learning and usage questions of minutes
+
+# Requirements
+
+- Python 3.5.2+
+- API powered by [Sanic](https://github.com/channelcat/sanic)
+- [Redis](https://redis.io/) as a database
+
 # Installation
 
-There are two main ways to run the app.
+There are few ready-to-go ways to run the Pieter:
+
+### Containers
+
+#### Easy way
 
-Containerized with [docker-compose](https://docs.docker.com/compose/):
+Containerized with [docker-compose](https://docs.docker.com/compose/)
 
 ```
 docker-compose up
 ```
 
+#### Flexible way
+
+Run in containers isolated in private network:
+
+```
+docker network create pieter
+
+docker run -d --name redis \
+    -p 6379:6379 \
+    --network pieter \
+    -v $(pwd)/data:/data \
+    redis
+
+docker run -d --name pieter-ci \
+    -p 8000:8000 \
+    --network pieter \
+    agrrh/pieter-ci:stable
+```
+
+You could also customize env variables pointing API on which address/port to listen and where to seek for DB, these are defaults:
+
+```
+PIETER_DB_HOST=redis
+PIETER_DB_PORT=6379
+PIETER_API_HOST=0.0.0.0
+PIETER_API_PORT=8000
+```
+
+### Manual
+
 Run locally for test purposes, development or containers-free setup:
 
 ```
-docker run -d -p 6379:6379 -v $(pwd)/data:/data redis
+# First, run redis in your way of choice
 
 # source local.env
 export PIETER_DB_HOST=127.0.0.1
@@ -24,12 +84,6 @@ export PIETER_API_PORT=8000
 http http://127.0.0.1:8000/
 ```
 
-# Requirements
-
-- Python 3.5.2+
-- API powered by [Sanic](https://github.com/channelcat/sanic)
-- [Redis](https://redis.io/) as a database
-
 All required libraries could be installed with:
 
 ```

ROADMAP.md

@@ -10,12 +10,15 @@ I guess it's OK to mention project in my CV now.
 Making it awesome.
 
 - [ ] Build UI
-- [ ] Add stdout/stderr streams
+- [ ] Accept GitHub hooks
+- [ ] Stream stdout/stderr?
 
 ### 0.3
 
 Start being proud.
 
+- [ ] Write docs on production usage
+- [ ] Provide common recipes examples
 - [x] Use some lightweight storage (redis, etcd, ...)
 - [x] Move repos out of `data.yml` and get rid of this file
 - [ ] Start testing it

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - redis
 
   redis:
-    image: redis:alpine
+    image: redis:3-alpine
     volumes:
       - data:/data
 

lib/api.py

@@ -142,7 +142,6 @@ async def scenario_actions(request, repo_name, scenario_name):
                     job.scenario = scenario.name
                     job.load()
                     await job.execute(scenario.data)
-                    job.save()
 
                     result = response.json(job.dump(), status=201)
 
@@ -153,7 +152,7 @@ async def job_actions(request, job_name):
             job = Job(self.db)
             job.load(job_name)
             if request.method == 'GET':
-                if job.rc is None:
+                if job.state is None:
                     result = response.json(None, status=404)
                 else:
                     result = response.json(job.dump())

lib/db_redis.py

@@ -6,9 +6,11 @@ class Database(object):
     def __init__(self, host='127.0.0.1', port=6379):
         self.handler = redis.StrictRedis(host=host, port=port, db=0)
 
-    def create(self, prefix, index, data):
+    def create(self, prefix, index, data, ttl=None):
         name = '_'.join((prefix, index))
         if self.handler.set(name, json.dumps(data)):
+            if ttl is not None:
+                self.handler.expire(name, ttl)
             return data
         return False
 

lib/job.py

@@ -9,6 +9,9 @@ class Job(object):
     def __init__(self, db, name=None):
         self.db = db
 
+        # Event loop
+        self.loop = None
+
         self.__build()
         self.name = str(uuid.uuid4())
 
@@ -24,28 +27,34 @@ def __build(self):
         self.stderr = None
         self.rc = None
 
+    async def background(self, script_path):
+        """Run process in background."""
+        process = await asyncio.create_subprocess_exec(script_path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+        stdout, stderr = await process.communicate()
+
+        self.time_done = int(time.time())
+        self.stdout = stdout.decode()
+        self.stderr = stderr.decode()
+        self.rc = process.returncode
+
     async def execute(self, scenario_data):
+        """Prepare and fire off a job."""
         self.state = 'running'
         self.time_start = int(time.time())
 
-        job_home_path = '/'.join(('', 'tmp', self.name))
+        self.save()
+
+        job_home_path = '/'.join(('jobs', self.name))
         job_script_path = '/'.join((job_home_path, self.scenario))
 
         os.mkdir(job_home_path, 0o755)
         with open(job_script_path, 'w') as fp:
             fp.write(scenario_data)
         os.chmod(job_script_path, 0o755)
 
-        process = await asyncio.create_subprocess_exec(job_script_path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
-        stdout, stderr = await process.communicate()
-
-        self.time_done = int(time.time())
-        self.stdout = stdout.decode()
-        self.stderr = stderr.decode()
-        self.rc = process.returncode
-
-        # FIXME uncomment following line if previous tasks runs in background
-        # self.save()
+        loop = asyncio.get_event_loop()
+        task = loop.create_task(self.background(job_script_path))
+        task.add_done_callback(self.save)
 
     def load(self, name=None):
         """Populate properties with values from DB."""
@@ -73,9 +82,9 @@ def load(self, name=None):
 
         return True
 
-    def save(self):
+    def save(self, *args):
         """Write object to database."""
-        return self.db.update('job', self.name, self.dump())
+        return self.db.update('job', self.name, self.dump(), ttl=3600)
 
     def dump(self):
         """Provide object as dict."""

misc/cleanup_jobs_dir.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+find ./jobs/* -type d | xargs -n1 rm -rf

misc/test_script.sh

@@ -2,5 +2,5 @@
 
 echo "Started: $(date +%s)"
 echo "Some payload here"
-sleep 2
+sleep 30
 echo "Done: $(date +%s)"