Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/zalando/spilo
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexeyklyukin
alexeyklyukin committed Apr 29, 2015
2 parents 30ca026 + 0364480 commit aa89a47
Show file tree
Hide file tree
Showing 8 changed files with 135 additions and 205 deletions.
67 changes: 27 additions & 40 deletions docs/DESIGN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Architecture: VPC
| |
| etcd |
| O DNS Service Record (SRV) |
| +-----> O O +-------> _etcd+server._tcp.example.com |
| +-----> O O +-------> _etcd-server._tcp.example.com |
| | O O<+ |
| | ^ | |
| | | | +-----+ |
Expand Down Expand Up @@ -62,50 +62,37 @@ The results of its discovery will also be stored in etcd, so other tools can eas
Architecture: PostgreSQL High Available cluster
================

+-----------------------+
| Elastic Load Balancer |
+-----------+-----------+
|
+-------+----------------------------+
EC2 Instance | EC2 Instance |
+--------------------------------+ +--------------------------------+
| | | | | |
| +--------------+ | | | +--------------+ | |
| | | | | | | | | |
| | etcd (proxy) | | | | | etcd (proxy) | | |
| | +----+ | | | | +----+ | |
| +--^-----------+ | | | | +--^-----------+ | | |
| | | | | | | | | |
| | | | | | | | | |
| | | | | | | | | |
| +--v-------+ +-v----v--+ | | +--v-------+ +-v---v---+ |
| | | | | | | | | | | |
| | Governor | | HAProxy | | | | Governor | | HAProxy | |
| | | | | | | | | | | |
| +--+-------+ +-+-------+ | | +--+-------+ +-+-------+ |
| | | | | | | |
| | | | | | | |
| | | | | | | |
| +---v--------+ | | | +---v--------+ | |
| | PostgreSQL | | | | | PostgreSQL | | |
| +------------+ | | | +------------+ | |
| | Slave | +----------------> Master <-------+ |
| +------------+ | | +------------+ |
| | | | | | | |
| +------------+ | | +------------+ |
| | | |
+--------------------------------+ +--------------------------------+


Healthcheck +-----------------------+
+-----------> Elastic Load Balancer <---+
| +-------------+---------+ |
| | |
| | +-------+
| | |
EC2 |nstance | | EC2 Instance
+----------------------------------+ | +----------------------------------+
| | | | | | |
| | | | | | |
| +--------+-+ +--------------+ | | | +-+--------+ +--------------+ |
| | | | | | | | | | | | |
| | Governor <----> etcd (proxy) | | | | | Governor <----> etcd (proxy) | |
| | | | | | | | | | | | |
| +--------^-+ +--------------+ | | | +--------^-+ +--------------+ |
| | | | | | |
| | | | | | |
| +-v----------+ | | | +-v----------+ |
| | PostgreSQL | | | | | PostgreSQL | |
| +------------+ | | | +------------+ |
| | Master <--------------+ | | Slave | |
| +------------+ | | +------------+ |
| | | | | | | |
| +------------+ | | +------------+ |
| | | |
+----------------------------------+ +----------------------------------+

etcd proxy
----
We assume a Higly Available etcd-cluster to be available for spilo when it is bootstrapped; we will use a etcd-proxy running on localhost to be a bridge between a HA-cluster member and the etcd-cluster.

HAProxy
----
As Yx

Governor
----

Expand Down
3 changes: 0 additions & 3 deletions docs/components/HAProxy.md
View file

This file was deleted.

6 changes: 3 additions & 3 deletions etcd-cluster-appliance/etcd-cluster.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ SenzaComponents:
2379: 2379
2380: 2380
runtime: Docker
source: pierone.stups.zalan.do/acid/etcd-cluster:{{Arguments.ImageVersion}}
source: {{Arguments.DockerImage}}
environment:
HOSTED_ZONE: '{{Arguments.HostedZone}}'
Type: Senza::TaupageAutoScalingGroup
Expand All @@ -24,8 +24,8 @@ SenzaInfo:
Parameters:
- HostedZone:
Description: AWS Hosted Zone to work with
- ImageVersion:
Description: Docker image version of etcd-cluster.
- DockerImage:
Description: Docker image of etcd-cluster.
StackName: etcd-cluster
Resources:
EtcdRole:
Expand Down
2 changes: 1 addition & 1 deletion etcd-cluster-appliance/etcd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ def members_changed(self):
return changed

def cluster_unhealthy(self):
process = subprocess.Popen([self.manager.ETCD_BINARY + 'ctl'], stdout=subprocess.PIPE)
process = subprocess.Popen([self.manager.ETCD_BINARY + 'ctl', 'cluster-health'], stdout=subprocess.PIPE)
ret = any([True for line in process.stdout if 'is unhealthy' in line])
process.wait()
return ret
Expand Down
87 changes: 13 additions & 74 deletions etcd-cluster-appliance/tests/test_etcd_cluster.py
View file
Original file line number Diff line number Diff line change
@@ -1,61 +1,13 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import boto.ec2
import unittest
import requests

from etcd import EtcdCluster, EtcdManager
from boto.ec2.instance import Instance


class MockResponse:

def __init__(self):
self.status_code = 200
self.content = '{}'

def json(self):
return json.loads(self.content)


def requests_post(url, **kwargs):
response = MockResponse()
data = json.loads(kwargs['data'])
if data['peerURLs'][0] in ['http://127.0.0.2:2380', 'http://127.0.0.3:2380']:
response.status_code = 201
response.content = '{{"id":"ifoobar","name":"","peerURLs":["{}"],"clientURLs":[""]}}'.format(data['peerURLs'
][0])
else:
response.status_code = 403
return response


def requests_get(url, **kwargs):
response = MockResponse()
response.content = \
'{"leaderInfo":{"leader":"ifoobari1"}, "members":[{"id":"ifoobari1","name":"i-deadbeef1","peerURLs":["http://127.0.0.1:2380"],"clientURLs":["http://127.0.0.1:2379"]},{"id":"ifoobari2","name":"i-deadbeef2","peerURLs":["http://127.0.0.2:2380"],"clientURLs":["http://127.0.0.2:2379"]},{"id":"ifoobari3","name":"i-deadbeef3","peerURLs":["http://127.0.0.3:2380"],"clientURLs":["ttp://127.0.0.3:2379"]},{"id":"ifoobari4","name":"i-deadbeef4","peerURLs":["http://127.0.0.4:2380"],"clientURLs":[]}]}'
return response


def requests_delete(url, **kwargs):
response = MockResponse()
response.status_code = 204
return response


def generate_instance(id, ip):
i = Instance()
i.id = id
i.private_ip_address = ip
i.private_dns_name = 'ip-{}.eu-west-1.compute.internal'.format(ip.replace('.', '-'))
i.tags = {'aws:cloudformation:stack-name': 'etc-cluster', 'aws:autoscaling:groupName': 'etc-cluster-postgres'}
return i


def manager_get_autoscaling_members():
return [generate_instance('i-deadbeef1', '127.0.0.1'), generate_instance('i-deadbeef2', '127.0.0.2'),
generate_instance('i-deadbeef3', '127.0.0.3')]
from test_etcd_manager import requests_post, requests_delete, requests_get, boto_ec2_connect_to_region


class TestEtcdCluster(unittest.TestCase):
Expand All @@ -68,8 +20,8 @@ def set_up(self):
requests.post = requests_post
requests.get = requests_get
requests.delete = requests_delete
boto.ec2.connect_to_region = boto_ec2_connect_to_region
self.manager = EtcdManager()
self.manager.get_autoscaling_members = manager_get_autoscaling_members
self.manager.instance_id = 'i-deadbeef3'
self.manager.region = 'eu-west-1'
self.cluster = EtcdCluster(self.manager)
Expand Down Expand Up @@ -104,28 +56,15 @@ def test_register_me(self):
'-initial-cluster-state',
'existing',
])

def test_initialize_new_cluster(self):
self.assertEqual(self.cluster.initialize_new_cluster(), [
'-name',
'i-deadbeef3',
'--data-dir',
'data',
'-listen-peer-urls',
'http://0.0.0.0:2380',
'-initial-advertise-peer-urls',
'http://127.0.0.3:2380',
'-listen-client-urls',
'http://0.0.0.0:2379',
'-advertise-client-urls',
'http://127.0.0.3:2379',
'-initial-cluster',
'i-deadbeef1=http://127.0.0.1:2380,i-deadbeef2=http://127.0.0.2:2380,i-deadbeef3=http://127.0.0.3:2380,i-deadbeef4=http://127.0.0.4:2380'
,
'-initial-cluster-token',
'etc-cluster',
'-initial-cluster-state',
'new',
])
self.cluster.me.id = 'ifoobari7'
self.assertRaises(Exception, self.cluster.register_me)
self.cluster.me.client_urls = []
self.cluster.me.id = ''
self.cluster.me.addr = '127.0.0.4'
self.assertRaises(Exception, self.cluster.register_me)
self.cluster.leader = None
self.assertRaises(Exception, self.cluster.register_me)
self.cluster.accessible_member = None
self.assertEqual(self.cluster.register_me()[17], 'new')


87 changes: 55 additions & 32 deletions etcd-cluster-appliance/tests/test_etcd_housekeeper.py
View file
Original file line number Diff line number Diff line change
@@ -1,55 +1,63 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import boto
import unittest
import requests
import subprocess

from etcd import EtcdManager, HouseKeeper
from boto.ec2.instance import Instance
from boto.route53.record import Record

from test_etcd_manager import boto_ec2_connect_to_region, requests_get, requests_delete, MockResponse

class MockResponse:

def __init__(self):
self.status_code = 200
self.content = '{}'
def requests_put(url, **kwargs):
response = MockResponse()
response.status_code = 201
return response

def json(self):
return json.loads(self.content)

class MockZone:

def requests_get(url, **kwargs):
response = MockResponse()
response.content = \
'{"leaderInfo":{"leader":"ifoobari1"}, "members":[{"id":"ifoobari1","name":"i-deadbeef1","peerURLs":["http://127.0.0.1:2380"],"clientURLs":["http://127.0.0.1:2379"]},{"id":"ifoobari2","name":"i-deadbeef2","peerURLs":["http://127.0.0.2:2380"],"clientURLs":["http://127.0.0.2:2379"]},{"id":"ifoobari3","name":"i-deadbeef3","peerURLs":["http://127.0.0.3:2380"],"clientURLs":["ttp://127.0.0.3:2379"]},{"id":"ifoobari4","name":"i-deadbeef4","peerURLs":["http://127.0.0.4:2380"],"clientURLs":[]}]}'
return response
def __init__(self, name):
self.name = name

def get_records(self):
if self.name != 'test.':
return []
r = Record()
r.name = '_etcd-server._tcp.cluster.' + self.name
r.type = 'SRV'
return [r]

def requests_put(url, **kwargs):
response = MockResponse()
response.status_code = 201
return response
def add_record(self, type, name, value):
pass

def update_record(self, old, new_value):
pass

def requests_delete(url, **kwargs):
response = MockResponse()
response.status_code = 204
return response

class MockRoute53Connection:

def get_zone(self, zone):
return (None if zone == 'bla' else MockZone(zone))

def generate_instance(id, ip):
i = Instance()
i.id = id
i.private_ip_address = ip
i.private_dns_name = 'ip-{}.eu-west-1.compute.internal'.format(ip.replace('.', '-'))
i.tags = {'aws:cloudformation:stack-name': 'etc-cluster', 'aws:autoscaling:groupName': 'etc-cluster-postgres'}
return i

def boto_route53_connect_to_region(region):
return MockRoute53Connection()

def manager_get_autoscaling_members():
return [generate_instance('i-deadbeef1', '127.0.0.1'), generate_instance('i-deadbeef2', '127.0.0.2'),
generate_instance('i-deadbeef3', '127.0.0.3')]

class Popen:

def __init__(self, args, **kwargs):
if args[1] != 'cluster-health':
raise Exception()
self.stdout = ['cluster is healthy', 'member 15a694aa6a6003f4 is healthy',
'member effbc38ed2b11107 is unhealthy']

def wait(self):
pass


class TestHouseKeeper(unittest.TestCase):
Expand All @@ -62,7 +70,10 @@ def set_up(self):
requests.get = requests_get
requests.put = requests_put
requests.delete = requests_delete
boto.ec2.connect_to_region = boto_ec2_connect_to_region
boto.route53.connect_to_region = boto_route53_connect_to_region
self.manager = EtcdManager()
self.manager.get_my_instace()
self.manager.instance_id = 'i-deadbeef3'
self.manager.region = 'eu-west-1'
self.keeper = HouseKeeper(self.manager, 'test.')
Expand All @@ -78,7 +89,19 @@ def test_acquire_lock(self):
self.assertEqual(self.keeper.acquire_lock(), True)

def test_remove_unhealthy_members(self):
autoscaling_members = manager_get_autoscaling_members()
autoscaling_members = self.manager.get_autoscaling_members()
self.assertEqual(self.keeper.remove_unhealthy_members(autoscaling_members), None)

def test_update_srv_record(self):
autoscaling_members = self.manager.get_autoscaling_members()
self.assertEqual(self.keeper.update_srv_record(autoscaling_members), None)
self.keeper.hosted_zone = 'bla'
self.assertEqual(self.keeper.update_srv_record(autoscaling_members), None)
self.keeper.hosted_zone = 'test2'
self.assertEqual(self.keeper.update_srv_record(autoscaling_members), None)

def test_cluster_unhealthy(self):
subprocess.Popen = Popen
self.assertEqual(self.keeper.cluster_unhealthy(), True)


0 comments on commit aa89a47

Please sign in to comment.