Skip to content
Permalink
Browse files

Merge branch 'release/v1.0.1'

  • Loading branch information...
etienne-napoleone committed Jan 4, 2019
2 parents 69e12ef + 0a9c18b commit 521c5a5324864883d9e68429a5f09e175e6d1916
Showing with 431 additions and 42 deletions.
  1. +4 βˆ’2 .travis.yml
  2. +1 βˆ’3 Dockerfile
  3. +37 βˆ’16 README.md
  4. +1 βˆ’1 goutte/__init__.py
  5. +42 βˆ’16 goutte/main.py
  6. +2 βˆ’2 poetry.lock
  7. +1 βˆ’1 pyproject.toml
  8. +63 βˆ’0 tests/mock.py
  9. +280 βˆ’1 tests/test_goutte.py
@@ -11,8 +11,8 @@ install:
script:
- flake8 .
- coverage run --source goutte -m pytest -v
# - coverage report
# - coveralls
- coverage report
- coveralls

before_deploy:
- poetry build
@@ -32,3 +32,5 @@ deploy:
on:
tags: true
python: 3.6
after_deploy:
- curl -XPOST $DOCKER_HOOK
@@ -2,11 +2,9 @@ FROM python:3.6-alpine

RUN pip3 install goutte

ENV GOUTTE_CONFIG goutte.yml
ENV GOUTTE_CONFIG goutte.toml
ENV GOUTTE_DO_TOKEN ''

WORKDIR /goutte

ENTRYPOINT ["goutte"]

CMD ["$GOUTTE_CONFIG", "$GOUTTE_DO_TOKEN"]
@@ -1,17 +1,15 @@
# goutte
# goutte <a href="https://travis-ci.org/tomochain/goutte"><img align="right" src="https://travis-ci.org/tomochain/goutte.svg?branch=develop"></a>
DigitalOcean doesn't propose any way of automating snapshots.
There are [some SaaS](https://snapshooter.io/) that can take care of it but paying to execute some API requests seemed a bit off.

That's why we developed a simple script which you can run with cron jobs or in CI services like Travis for free.
That's why we developed a simple script which can run with cron jobs or in CI services like Travis for free.
We use it daily to manage [our backups](https://github.com/tomochain/backups).

## TODO
- [x] Configuration from a single TOML file
- [x] Droplets snapshots
- [x] Droplets snapshots pruning
- [x] Volume snapshots
- [x] Volume snapshots pruning
- [ ] Slack alerting
- [ ] Add droplets and volumes by tag
It includes:
- Snapshoting droplets
- Snapshoting volumes
- Retention policy
- Pruning snapshots

## Requirements
- Python ^3.6
@@ -68,7 +66,7 @@ Options:
Running "snapshot only" for a configuration file containing one droplet and one volume:
```bash
$ goutte goutte.toml $do_token --only snapshot
13:32:48 - INFO - Starting goutte v1.0.0
13:32:48 - INFO - Starting goutte v1.0.1
13:32:52 - INFO - sgp1-website-01 - Snapshot (goutte-sgp1-website-01-20181220-56bde)
13:32:59 - INFO - sgp1-mariadb-01 - Snapshot (goutte-sgp1-mariadb-01-20181220-3673d)
```
@@ -84,10 +82,33 @@ docker run \
tomochain:goutte
```

## Automating
You can easily automate it via cron job or by leveraging free CI tools like Travis.
We provided and example travis configuration in `travis.example.yml`.
## Automating with Travis
You can easily automate it via cron job but the easiest way would be by leveraging free CI tools like Travis.

You just need to set the environment variables on the Travis website and schedule it with the frequency of your backups.
1. You can create a repo which contains your `goutte.toml` configuration and the following travis file `.travis.yml` :

TODO
```yml
language: python
python: 3.6
install:
- pip install goutte
script:
- goutte goutte.toml # Don't forget to set GOUTTE_DO_TOKEN in Travis config
```

2. Enable the repo in Travis and then go to the configuration
3. Add the environment variable GOUTTE_DO_TOKEN with the value of your DigitalOcean API key
4. Enable daily cron job
5. You're good to go, goutte will run everyday and take care of the snapshots.

**Note**: You can have different retentions for different volumes by having multiple configurations.
```yml
# ...
script:
- goutte 10days.toml
- goutte 1day.toml
```

You can see how we set it up for ourself [here](https://github.com/tomochain/backups).
@@ -1,6 +1,6 @@
import colorlog

__version__ = '1.0.0'
__version__ = '1.0.1'

handler = colorlog.StreamHandler()
handler.setFormatter(colorlog.ColoredFormatter(
@@ -12,6 +12,7 @@

log = colorlog.getLogger(__name__)
token = None
error = 0


@click.command(help='DigitalOcean snapshots automation.')
@@ -35,16 +36,17 @@ def entrypoint(config: click.File, do_token: str, only: str,
log.debug(f'Will only {only}')
_process_droplets(conf, only)
_process_volumes(conf, only)
sys.exit(error)


def _load_config(config: click.File) -> Dict[str, Dict]:
"""Return a config dict from a toml config file"""
try:
# TODO check minimum validity (retention)
log.debug('Loading config from {}'.format(config.name))
config = toml.load(config)
assert config['retention']
return config
conf = toml.load(config)
assert conf['retention']
return conf
except TypeError as e:
log.critical('Could not read conf {}: {}'.format(config.name, e))
sys.exit(1)
@@ -71,7 +73,7 @@ def _process_droplets(conf: Dict[str, Union[Dict[str, str], str]],
if only == 'snapshot' or not only:
_snapshot_droplet(droplet)
else:
log.warn('No matching droplet found')
log.warning('No matching droplet found')
except KeyError:
droplets = None
except KeyboardInterrupt:
@@ -93,9 +95,9 @@ def _process_volumes(conf: Dict[str, Union[Dict[str, str], str]],
if only == 'snapshot' or not only:
_snapshot_volume(volume)
else:
log.warn('No matching volume found')
log.warning('No matching volume found')
except KeyError:
volumes = None
pass
except KeyboardInterrupt:
log.critical('Received interuption signal')
sys.exit(1)
@@ -121,6 +123,7 @@ def _get_droplets(names: List[str]) -> List[digitalocean.Droplet]:

def _snapshot_droplet(droplet: digitalocean.Droplet) -> None:
"""Take a snapshot of a given droplet"""
global error
name = 'goutte-{}-{}-{}'.format(
droplet.name,
date.today().strftime('%Y%m%d'),
@@ -129,20 +132,26 @@ def _snapshot_droplet(droplet: digitalocean.Droplet) -> None:
droplet.take_snapshot(name)
log.info(f'{droplet.name} - Snapshot ({name})')
except digitalocean.baseapi.TokenError as e:
log.error(f'Token not valid: {e}')
log.error(f'Token not valid: {e}.')
error = 1
except digitalocean.baseapi.DataReadError as e:
log.error(f'Could not read response: {e}')
log.error(f'Could not read response: {e}.')
error = 1
except digitalocean.baseapi.JSONReadError as e:
log.error(f'Could not parse json: {e}')
log.error(f'Could not parse json: {e}.')
error = 1
except digitalocean.baseapi.NotFoundError as e:
log.error(f'Ressource not found: {e}')
log.error(f'Ressource not found: {e}.')
error = 1
except Exception as e:
log.error(f'Unexpected exception: {e}')
log.error(f'Unexpected exception: {e}.')
error = 1


def _prune_droplet_snapshots(droplet: digitalocean.Droplet,
retention: int) -> None:
"""Prune goutte snapshots if tmore than the configured retention time"""
global error
try:
all_snapshots = _order_snapshots([
digitalocean.Snapshot.get_object(
@@ -159,14 +168,19 @@ def _prune_droplet_snapshots(droplet: digitalocean.Droplet,
snapshot.destroy()
except digitalocean.baseapi.TokenError as e:
log.error(f'Token not valid: {e}.')
error = 1
except digitalocean.baseapi.DataReadError as e:
log.error(f'Could not read response: {e}.')
error = 1
except digitalocean.baseapi.JSONReadError as e:
log.error(f'Could not parse json: {e}.')
error = 1
except digitalocean.baseapi.NotFoundError as e:
log.error(f'Ressource not found: {e}.')
error = 1
except Exception as e:
log.error(f'Unexpected exception: {e}.')
error = 1


def _get_volumes(names: List[str]) -> List[digitalocean.Volume]:
@@ -189,6 +203,7 @@ def _get_volumes(names: List[str]) -> List[digitalocean.Volume]:

def _snapshot_volume(volume: digitalocean.Volume) -> None:
"""Take a snapshot of a given volume"""
global error
name = 'goutte-{}-{}-{}'.format(
volume.name,
date.today().strftime('%Y%m%d'),
@@ -197,20 +212,26 @@ def _snapshot_volume(volume: digitalocean.Volume) -> None:
volume.snapshot(name)
log.info(f'{volume.name} - Snapshot ({name})')
except digitalocean.baseapi.TokenError as e:
log.error(f'Token not valid: {e}')
log.error(f'Token not valid: {e}.')
error = 1
except digitalocean.baseapi.DataReadError as e:
log.error(f'Could not read response: {e}')
log.error(f'Could not read response: {e}.')
error = 1
except digitalocean.baseapi.JSONReadError as e:
log.error(f'Could not parse json: {e}')
log.error(f'Could not parse json: {e}.')
error = 1
except digitalocean.baseapi.NotFoundError as e:
log.error(f'Ressource not found: {e}')
log.error(f'Ressource not found: {e}.')
error = 1
except Exception as e:
log.error(f'Unexpected exception: {e}')
log.error(f'Unexpected exception: {e}.')
error = 1


def _prune_volume_snapshots(volume: digitalocean.Volume,
retention: int) -> None:
"""Prune goutte snapshots if tmore than the configured retention time"""
global error
try:
all_snapshots = _order_snapshots(volume.get_snapshots())
snapshots = [snapshot for snapshot in all_snapshots
@@ -223,14 +244,19 @@ def _prune_volume_snapshots(volume: digitalocean.Volume,
snapshot.destroy()
except digitalocean.baseapi.TokenError as e:
log.error(f'Token not valid: {e}.')
error = 1
except digitalocean.baseapi.DataReadError as e:
log.error(f'Could not read response: {e}.')
error = 1
except digitalocean.baseapi.JSONReadError as e:
log.error(f'Could not parse json: {e}.')
error = 1
except digitalocean.baseapi.NotFoundError as e:
log.error(f'Ressource not found: {e}.')
error = 1
except Exception as e:
log.error(f'Unexpected exception: {e}.')
error = 1


def _order_snapshots(snapshots: List[digitalocean.Snapshot]
@@ -53,7 +53,7 @@ description = "Log formatting with colors!"
name = "colorlog"
optional = false
python-versions = "*"
version = "3.1.4"
version = "3.2.0"

[package.dependencies]
colorama = "*"
@@ -263,7 +263,7 @@ certifi = ["47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", "
chardet = ["84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"]
click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"]
colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"]
colorlog = ["418db638c9577f37f0fae4914074f395847a728158a011be2a193ac491b9779d", "8b234ebae1ba1237bc79c0d5f1f47b31a3f3e90c0b4c2b0ebdde63a174d3b97b"]
colorlog = ["31378a98b965c9f2bc5fb58c906e0e6d8d2922f6b8229c39903711da5b490fc2", "45e76dc65c0ed0e8c27175c00b18d92016dc58a6feff62e168819a2bca26df68"]
coverage = ["06123b58a1410873e22134ca2d88bd36680479fe354955b3579fb8ff150e4d27", "09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", "0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", "0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", "0d34245f824cc3140150ab7848d08b7e2ba67ada959d77619c986f2062e1f0e8", "10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", "1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", "1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", "258b21c5cafb0c3768861a6df3ab0cfb4d8b495eee5ec660e16f928bf7385390", "2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", "3ad59c84c502cd134b0088ca9038d100e8fb5081bbd5ccca4863f3804d81f61d", "447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", "46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", "4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", "510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", "5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", "5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", "5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", "6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", "6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", "71afc1f5cd72ab97330126b566bbf4e8661aab7449f08895d21a5d08c6b051ff", "7349c27128334f787ae63ab49d90bf6d47c7288c63a0a5dfaa319d4b4541dd2c", "77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", "828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", "859714036274a75e6e57c7bab0c47a4602d2a8cfaaa33bbdb68c8359b2ed4f5c", "85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", "869ef4a19f6e4c6987e18b315721b8b971f7048e6eaea29c066854242b4e98d9", "8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", "977e2d9a646773cc7428cdd9a34b069d6ee254fadfb4d09b3f430e95472f3cf3", "99bd767c49c775b79fdcd2eabff405f1063d9d959039c0bdd720527a7738748a", "a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", "aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", "ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", "b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", "bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", "c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", "d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", "d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", "da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", "ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", "ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"]
coveralls = ["ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d", "b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0"]
docopt = ["49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"]
@@ -1,6 +1,6 @@
[tool.poetry]
name = "goutte"
version = "1.0.0"
version = "1.0.1"
description = "DigitalOcean snapshot automation service"
readme = "README.md"
license = "GPL-3.0+"
@@ -0,0 +1,63 @@
def nothing(*args, **kwargs):
pass


class Snapshot:
def __init__(self, created_at=None, name=None, id=None):
self.created_at = created_at
self.name = name
self.id = id

def destroy(self):
pass

@staticmethod
def get_object(api_token=None, snapshot_id=None):
if snapshot_id == '1337':
return Snapshot(name=f'snapshot{snapshot_id}', id=snapshot_id,
created_at=f'{snapshot_id}')
else:
return Snapshot(name=f'goutte-snapshot{snapshot_id}',
id=snapshot_id, created_at=f'{snapshot_id}')


class Volume:
def __init__(self, name=None, snapshots=None, throw=None):
self.name = name
self.snapshots = snapshots
self.throw = throw

def get_snapshots(self):
return self.snapshots

def snapshot(self, name):
pass


class Droplet:
def __init__(self, name=None, snapshot_ids=None):
self.name = name
self.snapshot_ids = snapshot_ids

def take_snapshot(self, name):
pass


class Manager:
def __init__(self, token=None):
self.token = token

def get_all_volumes(self):
return [
Volume(name='testvol')
]

def get_all_droplets(self):
return [
Droplet(name='testdroplet')
]


class File:
def __init__(self, name=None):
self.name = name
Oops, something went wrong.

0 comments on commit 521c5a5

Please sign in to comment.
You can’t perform that action at this time.