Skip to content

Commit

Permalink
[IMP] clouder: Centralize and improve path/str handling
Browse files Browse the repository at this point in the history 
* Centralize key and clouder tmp dirs via `ClouderModel` method
* Centralize backup directory definitions in `ClouderModel`
* Harden path handling by using os.path.join
* Prefer string interpolation vs concat
* Fix use of `self.raise_error` for models that aren't inherited from `clouder.model`
* Prefer `ClouderError` in models not inherited from `clouder.model`
* Update model attribute declaration orders
* Use lambdas for defaults in `clouder.container`
* Environment prefix is required
  • Loading branch information
@lasley
lasley committed Oct 17, 2016
1 parent 252f8d2 commit c77c0c5
Show file tree
Hide file tree
Showing 10 changed files with 623 additions and 370 deletions.
39 changes: 23 additions & 16 deletions clouder/application.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@
#
##############################################################################


from openerp import models, fields, api
from datetime import datetime
import os.path
import re

from openerp import models, fields, api

from . import model


Expand Down Expand Up @@ -154,6 +155,10 @@ class ClouderApplication(models.Model):
"""

_name = 'clouder.application'
_sql_constraints = [
('name_uniq', 'unique(name)', 'Name must be unique!'),
]
_order = 'sequence, code'

name = fields.Char('Name', required=True)
code = fields.Char('Code', required=True)
Expand Down Expand Up @@ -226,35 +231,43 @@ def full_archivepath(self):
Property returning the full path to the archive
in the archive container
"""
return self.env['clouder.model'].archive_path + '/' \
+ self.type_id.name + '-' + self.code
return os.path.join(
self.env['clouder.model'].archive_path,
self.type_id.name, self.code,
)

@property
def full_hostpath(self):
"""
Property returning the full path to the archive
in the hosting system.
"""
return self.env['clouder.model'].services_hostpath + '/' \
+ self.type_id.name + '-' + self.code
return os.path.join(
self.env['clouder.model'].services_hostpath,
self.type_id.name, self.code,
)

@property
def full_localpath(self):
"""
Property returning the full path to the instance
in the destination container
"""
return self.type_id.localpath and self.type_id.localpath + '/' \
+ self.type_id.name + '-' + self.code or ''
return os.path.join(
self.type_id.localpath and self.type_id.localpath,
self.type_id.name, self.code or '',
)

@property
def computed_version(self):
"""
Property returning the name of the application version
with the current date.
"""
return self.current_version + '.' \
+ datetime.now().strftime('%Y%m%d.%H%M%S')
return '%s.%s' % (
self.current_version,
datetime.now().strftime('%Y%m%d.%H%M%S'),
)

@property
def options(self):
Expand Down Expand Up @@ -285,12 +298,6 @@ def links(self):
links[link.name.code] = link
return links

_sql_constraints = [
('name_uniq', 'unique(name)', 'Name must be unique!'),
]

_order = 'sequence, code'

@api.multi
@api.constrains('code', 'admin_name', 'admin_email')
def _check_forbidden_chars_credentials_code(self):
Expand Down
25 changes: 13 additions & 12 deletions clouder/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ class ClouderBase(models.Model):

_name = 'clouder.base'
_inherit = ['clouder.model']
_sql_constraints = [
('name_uniq', 'unique (name,domain_id)',
'Name must be unique per domain !')
]

name = fields.Char('Name', required=True)
domain_id = fields.Many2one('clouder.domain', 'Domain name', required=True)
Expand Down Expand Up @@ -153,8 +157,10 @@ def fullname(self):
"""
Property returning the full name of the base.
"""
return self.application_id.fullcode + '-' + \
self.fulldomain.replace('.', '-')
return '%s-%s' % (
self.application_id.fullcode,
self.fulldomain.replace('.', '-'),
)

@property
def fullname_(self):
Expand All @@ -171,7 +177,7 @@ def fulldomain(self):
"""
if self.is_root:
return self.domain_id.name
return self.name + '.' + self.domain_id.name
return '%s.%s' % (self.name, self.domain_id.name)

@property
def databases(self):
Expand All @@ -180,10 +186,10 @@ def databases(self):
"""
databases = {'single': self.fullname_}
if self.application_id.type_id.multiple_databases:
databases = {}
for database in \
self.application_id.type_id.multiple_databases.split(','):
databases[database] = self.fullname_ + '_' + database
dbs = self.application_id.type_id.multiple_databases.split(',')
databases = {
db: '%s_%s' % (self.fullname_, db) for db in dbs
}
return databases

@property
Expand Down Expand Up @@ -228,11 +234,6 @@ def links(self):
links[link.name.name.code] = link
return links

_sql_constraints = [
('name_uniq', 'unique (name,domain_id)',
'Name must be unique per domain !')
]

@api.multi
@api.constrains('name', 'admin_name', 'admin_email', 'poweruser_email')
def _check_forbidden_chars_credentials(self):
Expand Down
62 changes: 41 additions & 21 deletions clouder/clouder_runner_docker/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@
#
##############################################################################

from openerp import models, api, modules
import time
from datetime import datetime

import logging
import os.path
import time

from openerp import models, api, modules

_logger = logging.getLogger(__name__)


Expand All @@ -46,47 +48,65 @@ def build_image(

if not runner or runner.application_id.type_id.name == 'docker':

path = model.name + '-' + datetime.now().strftime('%Y%m%d.%H%M%S')
path = '%s-%s' % (
model.name, datetime.now().strftime('%Y%m%d.%H%M%S'),
)
if model._name == 'clouder.container':
name = path
else:
name = model.fullpath

if salt:
build_dir = '/srv/salt/containers/build_' + model.name
build_dir = os.path.join(
'/srv', 'salt', 'containers', 'build_%s' % model.name,
)
server = model.salt_master
else:
build_dir = '/tmp/' + name
build_dir = self.env['clouder.model']._get_directory_tmp(name)

server.execute(['rm', '-rf', build_dir])
server.execute(['mkdir', '-p', build_dir])

if self.type_id:
if self.type_id.name in \
['backup', 'salt-master', 'salt-minion']:
sources_path = \
modules.get_module_path('clouder') + '/sources'
if self.type_id.name in [
'backup', 'salt-master', 'salt-minion'
]:
sources_path = os.path.join(
modules.get_module_path('clouder'), 'sources',
)
else:
module_path = modules.get_module_path(
'clouder_template_' + self.type_id.name
'clouder_template_%s' % self.type_id.name
)
sources_path = module_path and os.path.join(
module_path, 'sources'
)
if sources_path and self.env['clouder.model'].local_dir_exist(
sources_path
):
server.send_dir(
sources_path, os.path.join(build_dir, 'sources'),
)
sources_path = module_path and module_path + '/sources'
if sources_path and self.env['clouder.model']\
.local_dir_exist(sources_path):
server.send_dir(sources_path, build_dir + '/sources')

docker_file = os.path.join(build_dir, 'Dockerfile')
server.execute([
'echo "' + self.computed_dockerfile.replace('"', r'\\"') +
'" >> ' + build_dir + '/Dockerfile'])
'echo "%s" >> "%s"' % (
self.computed_dockerfile.replace('"', r'\\"'),
docker_file,
),
])

if expose_ports:
server.execute([
'echo "' + 'EXPOSE ' + ' '.join(expose_ports) +
'" >> ' + build_dir + '/Dockerfile'])
'echo "EXPOSE %s" >> "%s"' % (
' '.join(expose_ports), docker_file,
),
])

if not salt:
server.execute(
['docker', 'build', '--pull', '-t', name, build_dir])
server.execute([
'docker', 'build', '--pull', '-t', name, build_dir,
])
server.execute(['rm', '-rf', build_dir])

return name
Expand Down
23 changes: 15 additions & 8 deletions clouder/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@
#
##############################################################################

from openerp import models, fields, api
from datetime import datetime
import os.path
import re

from datetime import datetime
from openerp import models, fields, api


class ClouderConfigBackupMethod(models.Model):
Expand Down Expand Up @@ -97,20 +98,26 @@ def save_all_exec(self):
on all containers and bases.
"""
self = self.with_context(save_comment='Save before upload_save')
backup_dir = os.path.join(self.BACKUP_DIR, 'bup')

backups = self.env['clouder.container'].search(
[('application_id.type_id.name', '=', 'backup')])
for backup in backups:
backup.execute(['export BUP_DIR=/opt/backup/bup;', 'bup', 'fsck',
'-r'], username="backup")
backup.execute(
['export BUP_DIR="%s";' % backup_dir, 'bup', 'fsck', '-r'],
username="backup",
)
# http://stackoverflow.com/questions/1904860/
# how-to-remove-unreferenced-blobs-from-my-git-repo
# https://github.com/zoranzaric/bup/tree/tmp/gc/Documentation
# https://groups.google.com/forum/#!topic/bup-list/uvPifF_tUVs
backup.execute(['git', 'gc', '--prune=now'],
path='/opt/backup/bup', username="backup")
backup.execute(['export BUP_DIR=/opt/backup/bup;', 'bup', 'fsck',
'-g'], username="backup")
backup.execute(
['git', 'gc', '--prune=now'], backup_dir, username="backup",
)
backup.execute(
['export BUP_DIR="%s";' % backup_dir, 'bup', 'fsck', '-g'],
username="backup",
)

containers = self.env['clouder.container'].search(
[('autosave', '=', True)])
Expand Down
Loading

0 comments on commit c77c0c5

Please sign in to comment.