Skip to content

Commit

Permalink
incremental backup: exclude repo folder
Browse files Browse the repository at this point in the history
  • Loading branch information
@usmannasir
usmannasir committed Oct 28, 2019
1 parent 683ada6 commit 71c97e0
Show file tree
Hide file tree
Showing 7 changed files with 763 additions and 221 deletions.
408 changes: 240 additions & 168 deletions .idea/workspace.xml
View file

Large diffs are not rendered by default.

140 changes: 109 additions & 31 deletions IncBackups/IncBackupsControl.py
View file
Expand Up @@ -2,10 +2,12 @@
import os import os
import os.path import os.path
import sys import sys

sys.path.append('/usr/local/CyberCP') sys.path.append('/usr/local/CyberCP')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")


import django import django

try: try:
django.setup() django.setup()
except: except:
Expand All @@ -25,12 +27,16 @@
from plogical.dnsUtilities import DNS from plogical.dnsUtilities import DNS
from mailServer.models import Domains as eDomains from mailServer.models import Domains as eDomains
from random import randint from random import randint
import json
from django.shortcuts import HttpResponse

try: try:
from plogical.virtualHostUtilities import virtualHostUtilities from plogical.virtualHostUtilities import virtualHostUtilities
from plogical.mailUtilities import mailUtilities from plogical.mailUtilities import mailUtilities
except: except:
pass pass



class IncJobs(multi.Thread): class IncJobs(multi.Thread):


def __init__(self, function, extraArgs): def __init__(self, function, extraArgs):
Expand All @@ -51,29 +57,97 @@ def run(self):
self.createBackup() self.createBackup()
elif self.function == 'restorePoint': elif self.function == 'restorePoint':
self.restorePoint() self.restorePoint()
elif self.function == 'remoteRestore':
self.restorePoint()

def getRemoteBackups(self):
if self.jobid.destination == 'local':
path = '/home/%s/incbackup' % (self.website)
command = 'export RESTIC_PASSWORD=%s && restic -r %s snapshots' % (self.passwordFile, path)
return ProcessUtilities.outputExecutioner(command).split('\n')
elif self.jobid.destination[:4] == 'sftp':
path = '/home/backup/%s' % (self.website)
command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r %s:%s snapshots' % (
self.passwordFile, self.backupDestinations, path)
return ProcessUtilities.outputExecutioner(command).split('\n')
else:
path = '/home/%s/incbackup' % (self.website)
command = 'export RESTIC_PASSWORD=%s && restic -r %s snapshots' % (self.passwordFile, path)
return ProcessUtilities.outputExecutioner(command).split('\n')

def fetchCurrentBackups(self):
try:
self.website = self.extraArgs['website']
self.backupDestinations = self.extraArgs['backupDestinations']
self.passwordFile = self.extraArgs['password']

remotePath = '/home/backup/%s' % (self.website)

command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r %s:%s snapshots' % (
self.passwordFile, self.backupDestinations, remotePath)
result = ProcessUtilities.outputExecutioner(command).split('\n')

activator = 0
json_data = "["
checker = 0

for items in result:
if items.find('---------------') > -1:
if activator == 0:
activator = 1
continue
else:
activator = 0

if activator:
entry = items.split(' ')
logging.writeToFile(str(entry))

dic = {'id': entry[0],
'date': "%s %s" % (entry[2], entry[3]),
'host': entry[5],
'path': entry[-1]
}

if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)

json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)

except BaseException, msg:
logging.writeToFile(str(msg))

####


def getAWSData(self): def getAWSData(self):
key = self.backupDestinations.split('/')[-1] key = self.backupDestinations.split('/')[-1]
path = '/home/cyberpanel/aws/%s' % (key) path = '/home/cyberpanel/aws/%s' % (key)
secret = open(path, 'r').read() secret = open(path, 'r').read()
return key, secret return key, secret


def awsFunction(self, fType, backupPath = None, snapshotID = None, bType = None): def awsFunction(self, fType, backupPath=None, snapshotID=None, bType=None):
try: try:
if fType == 'backup': if fType == 'backup':
key, secret = self.getAWSData() key, secret = self.getAWSData()
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s backup %s --password-file %s' % ( command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s backup %s --password-file %s' % (
key, secret, self.website.domain, backupPath, self.passwordFile) key, secret, self.website.domain, backupPath, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)
snapShotid = result.split(' ')[-2] snapShotid = result.split(' ')[-2]
if bType == 'database': if bType == 'database':
newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (bType, backupPath.split('/')[-1].strip('.sql')), newSnapshot = JobSnapshots(job=self.jobid,
type='%s:%s' % (bType, backupPath.split('/')[-1].strip('.sql')),
snapshotid=snapShotid, snapshotid=snapShotid,
destination=self.backupDestinations) destination=self.backupDestinations)
else: else:
newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (bType, backupPath), snapshotid=snapShotid, newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (bType, backupPath),
destination=self.backupDestinations) snapshotid=snapShotid,
destination=self.backupDestinations)
newSnapshot.save() newSnapshot.save()
else: else:
self.backupDestinations = self.jobid.destination self.backupDestinations = self.jobid.destination
Expand All @@ -88,9 +162,9 @@ def awsFunction(self, fType, backupPath = None, snapshotID = None, bType = None)
logging.statusWriter(self.statusPath, "%s [88][5009]" % (str(msg)), 1) logging.statusWriter(self.statusPath, "%s [88][5009]" % (str(msg)), 1)
return 0 return 0


def localFunction(self, backupPath, type, restore = None): def localFunction(self, backupPath, type, restore=None):
if restore == None: if restore == None:
command = 'restic -r %s backup %s --password-file %s' % (self.repoPath, backupPath, self.passwordFile) command = 'restic -r %s backup %s --password-file %s --exclude %s' % (self.repoPath, backupPath, self.passwordFile, self.repoPath)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)
snapShotid = result.split(' ')[-2] snapShotid = result.split(' ')[-2]
Expand All @@ -107,11 +181,11 @@ def localFunction(self, backupPath, type, restore = None):
else: else:
repoLocation = '/home/%s/incbackup' % (self.website) repoLocation = '/home/%s/incbackup' % (self.website)
command = 'restic -r %s restore %s --target / --password-file %s' % ( command = 'restic -r %s restore %s --target / --password-file %s' % (
repoLocation, self.jobid.snapshotid, self.passwordFile) repoLocation, self.jobid.snapshotid, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)


def sftpFunction(self, backupPath, type, restore = None): def sftpFunction(self, backupPath, type, restore=None):
if restore == None: if restore == None:
remotePath = '/home/backup/%s' % (self.website.domain) remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % ( command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
Expand All @@ -132,7 +206,7 @@ def sftpFunction(self, backupPath, type, restore = None):
else: else:
repoLocation = '/home/backup/%s' % (self.website) repoLocation = '/home/backup/%s' % (self.website)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % ( command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (
self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile) self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)


Expand Down Expand Up @@ -160,7 +234,8 @@ def restoreDatabase(self):
else: else:
self.awsFunction('restore', '', self.jobid.snapshotid) self.awsFunction('restore', '', self.jobid.snapshotid)


if mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(self.jobid.type.split(':')[1].rstrip('.sql'), '/home/cyberpanel', 'dummy', 'dummy') == 0: if mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(self.jobid.type.split(':')[1].rstrip('.sql'),
'/home/cyberpanel', 'dummy', 'dummy') == 0:
raise BaseException raise BaseException


try: try:
Expand Down Expand Up @@ -189,7 +264,6 @@ def restoreEmail(self):
def reconstructWithMeta(self): def reconstructWithMeta(self):
try: try:



if self.jobid.destination == 'local': if self.jobid.destination == 'local':
self.localFunction('none', 'none', 1) self.localFunction('none', 'none', 1)
elif self.jobid.destination[:4] == 'sftp': elif self.jobid.destination[:4] == 'sftp':
Expand Down Expand Up @@ -219,7 +293,6 @@ def restorePoint(self):
logging.statusWriter(self.statusPath, message, 1) logging.statusWriter(self.statusPath, message, 1)
self.passwordFile = '/home/%s/%s' % (self.website, self.website) self.passwordFile = '/home/%s/%s' % (self.website, self.website)



## ##


if self.jobid.type[:8] == 'database': if self.jobid.type[:8] == 'database':
Expand Down Expand Up @@ -433,10 +506,11 @@ def backupData(self):
else: else:
self.awsFunction('backup', backupPath, '', 'data') self.awsFunction('backup', backupPath, '', 'data')


logging.statusWriter(self.statusPath, 'Data for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1) logging.statusWriter(self.statusPath,
'Data for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
return 1 return 1
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.backupData.223][5009]' % str(msg), 1) logging.statusWriter(self.statusPath, '%s. [IncJobs.backupData.223][5009]' % str(msg), 1)
return 0 return 0


def backupDatabases(self): def backupDatabases(self):
Expand All @@ -461,10 +535,11 @@ def backupDatabases(self):
try: try:
os.remove('/home/cyberpanel/%s.sql' % (items.dbName)) os.remove('/home/cyberpanel/%s.sql' % (items.dbName))
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'Failed to delete database: %s. [IncJobs.backupDatabases.456]' % str(msg), 1) logging.statusWriter(self.statusPath,
'Failed to delete database: %s. [IncJobs.backupDatabases.456]' % str(msg), 1)
return 1 return 1
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.backupDatabases.269][5009]' % str(msg), 1) logging.statusWriter(self.statusPath, '%s. [IncJobs.backupDatabases.269][5009]' % str(msg), 1)
return 0 return 0


def emailBackup(self): def emailBackup(self):
Expand All @@ -480,11 +555,11 @@ def emailBackup(self):
else: else:
self.awsFunction('backup', backupPath, '', 'email') self.awsFunction('backup', backupPath, '', 'email')



logging.statusWriter(self.statusPath,
logging.statusWriter(self.statusPath, 'Emails for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1) 'Emails for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
return 1 return 1
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.emailBackup.269][5009]' % str(msg), 1) logging.statusWriter(self.statusPath, '%s. [IncJobs.emailBackup.269][5009]' % str(msg), 1)
return 0 return 0


def metaBackup(self): def metaBackup(self):
Expand All @@ -500,11 +575,11 @@ def metaBackup(self):
else: else:
self.awsFunction('backup', backupPath, '', 'meta') self.awsFunction('backup', backupPath, '', 'meta')



logging.statusWriter(self.statusPath,
logging.statusWriter(self.statusPath, 'Meta for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1) 'Meta for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
return 1 return 1
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.metaBackup.269][5009]' % str(msg), 1) logging.statusWriter(self.statusPath, '%s. [IncJobs.metaBackup.269][5009]' % str(msg), 1)
return 0 return 0


def initiateRepo(self): def initiateRepo(self):
Expand All @@ -518,20 +593,23 @@ def initiateRepo(self):


elif self.backupDestinations[:4] == 'sftp': elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain) remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic init --repo %s:%s --password-file %s' % (self.backupDestinations, remotePath, self.passwordFile) command = 'export PATH=${PATH}:/usr/bin && restic init --repo %s:%s --password-file %s' % (
self.backupDestinations, remotePath, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)
else: else:
key,secret = self.getAWSData() key, secret = self.getAWSData()
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s init --password-file %s' % (key, secret, self.website.domain, self.passwordFile) command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s init --password-file %s' % (
key, secret, self.website.domain, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command) result = ProcessUtilities.outputExecutioner(command)
logging.statusWriter(self.statusPath, result, 1) logging.statusWriter(self.statusPath, result, 1)
return 1 return 1


logging.statusWriter(self.statusPath, 'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1) logging.statusWriter(self.statusPath,
'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
return 1 return 1
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.initiateRepo.47][5009]' % str(msg), 1) logging.statusWriter(self.statusPath, '%s. [IncJobs.initiateRepo.47][5009]' % str(msg), 1)
return 0 return 0


def createBackup(self): def createBackup(self):
Expand Down Expand Up @@ -562,7 +640,6 @@ def createBackup(self):
command = 'chmod 600 %s' % (self.passwordFile) command = 'chmod 600 %s' % (self.passwordFile)
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)



if self.initiateRepo() == 0: if self.initiateRepo() == 0:
return return


Expand All @@ -589,6 +666,7 @@ def createBackup(self):
command = 'rm -f %s' % (metaPathNew) command = 'rm -f %s' % (metaPathNew)
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
except BaseException, msg: except BaseException, msg:
logging.statusWriter(self.statusPath,'Failed to delete meta file: %s. [IncJobs.createBackup.591]' % str(msg), 1) logging.statusWriter(self.statusPath,
'Failed to delete meta file: %s. [IncJobs.createBackup.591]' % str(msg), 1)


logging.statusWriter(self.statusPath, 'Completed', 1) logging.statusWriter(self.statusPath, 'Completed', 1)

0 comments on commit 71c97e0

Please sign in to comment.