This repository has been archived by the owner on Aug 25, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
19 changed files
with
498 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
import logging | ||
from abc import abstractmethod | ||
|
||
|
||
class AbstractShouldBackupPredicate(object): | ||
|
||
@abstractmethod | ||
def test(self, big_query_table_metadata, table_entity): | ||
raise NotImplementedError | ||
|
||
@staticmethod | ||
def _is_possible_to_copy_table(big_query_table_metadata): | ||
if not big_query_table_metadata.table_exists(): | ||
logging.info('Table not found (404)') | ||
return False | ||
if not big_query_table_metadata.is_schema_defined(): | ||
logging.info('This table is without schema') | ||
return False | ||
if big_query_table_metadata.is_external_or_view_type(): | ||
return False | ||
return True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
import logging | ||
|
||
from src.backup.abstract_should_backup_predicate import \ | ||
AbstractShouldBackupPredicate | ||
|
||
|
||
class OnDemandShouldBackupPredicate(AbstractShouldBackupPredicate): | ||
|
||
def test(self, big_query_table_metadata, table_entity): | ||
if not self._is_possible_to_copy_table(big_query_table_metadata): | ||
return False | ||
|
||
if big_query_table_metadata.is_empty(): | ||
logging.info('This table is empty') | ||
|
||
logging.info( | ||
"Performing on-demand backup for %s:%s.%s$%s. " | ||
"It is performed without checking if table aready has up to date backup", | ||
table_entity.project_id, table_entity.dataset_id, | ||
table_entity.table_id, table_entity.partition_id) | ||
|
||
return True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
from src.backup.backup_process import BackupProcess | ||
from src.backup.on_demand.on_demand_should_backup_predicate import \ | ||
OnDemandShouldBackupPredicate | ||
from src.commons.big_query.big_query import BigQuery | ||
from src.commons.big_query.big_query_table_metadata import BigQueryTableMetadata | ||
from src.commons.exceptions import ParameterValidationException | ||
|
||
|
||
class PartitionedTableWithoutPartition(Exception): | ||
pass | ||
|
||
|
||
class OnDemandTableBackup(object): | ||
|
||
@staticmethod | ||
def start(table_reference): | ||
big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference(table_reference) | ||
|
||
if big_query_table_metadata.is_daily_partitioned() and not big_query_table_metadata.is_partition(): | ||
raise ParameterValidationException("Partition id is required for partitioned table in on-demand mode") | ||
|
||
BackupProcess(table_reference=table_reference, | ||
big_query=BigQuery(), | ||
big_query_table_metadata=big_query_table_metadata, | ||
should_backup_predicate=OnDemandShouldBackupPredicate()).start() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
import webapp2 | ||
|
||
from src.backup.on_demand.on_demand_table_backup import OnDemandTableBackup | ||
from src.commons.config.configuration import configuration | ||
from src.commons.table_reference import TableReference | ||
from src.commons.tasks import Tasks | ||
from src.commons.handlers.json_handler import JsonHandler | ||
|
||
|
||
class OnDemandTableBackupHandler(JsonHandler): | ||
def __init__(self, request=None, response=None): | ||
super(OnDemandTableBackupHandler, self).__init__(request, response) | ||
|
||
# now let's check if this task is not a retry of some previous (which | ||
# failed for some reason) if so - let's log when it hits the defined | ||
# mark so we can catch it on monitoring: | ||
Tasks.log_task_metadata_for(request=self.request) | ||
|
||
def get(self, project_id, dataset_id, table_id, partition_id=None): # nopep8 pylint: disable=R0201 | ||
table_reference = TableReference(project_id, dataset_id, | ||
table_id, partition_id) | ||
OnDemandTableBackup.start(table_reference) | ||
|
||
|
||
app = webapp2.WSGIApplication([ | ||
webapp2.Route('/tasks/backups/on_demand/table/<project_id:[^/]+>/<dataset_id:' | ||
'[^/]+>/<table_id:[^/]+>', OnDemandTableBackupHandler), | ||
webapp2.Route('/tasks/backups/on_demand/table/<project_id:[^/]+>/<dataset_id:' | ||
'[^/]+>/<table_id:[^/]+>/<partition_id:[^/]+>', | ||
OnDemandTableBackupHandler) | ||
], debug=configuration.debug_mode) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
{% extends "bbq-menu.html" %} | ||
{% block main %} | ||
|
||
<script type="text/javascript"> | ||
function OnSubmitForm() { | ||
var url = '/tasks/backups/on_demand/table/' + document.restoreTableForm.projectId.value + '/' | ||
+ document.restoreTableForm.datasetId.value + '/' + document.restoreTableForm.tableId.value | ||
if (document.restoreTableForm.partitionId.value) { | ||
url = url + '/' + document.restoreTableForm.partitionId.value | ||
} | ||
|
||
|
||
var xhttp = new XMLHttpRequest(); | ||
xhttp.open("GET", url, true); | ||
xhttp.setRequestHeader("request_correlation_id", Date.now()) | ||
xhttp.timeout = 1000 * 3600; | ||
|
||
xhttp.onload = function () { | ||
document.getElementById("response").innerHTML = 'Http response: '+ xhttp.status + '<br>' + xhttp.response | ||
}; | ||
xhttp.ontimeout = function (e) { | ||
document.getElementById("response").innerHTML = "Request timeout: " + e; | ||
}; | ||
|
||
xhttp.send(); | ||
return false; | ||
} | ||
</script> | ||
<h2>On-demand table backup</h2> | ||
<p class="lead"> | ||
BBQ will schedule on-demand backup of given source table. <br> | ||
On-demand flow doesn't check if table already has up to date backup<br> | ||
|
||
Note that successfully scheduled on-demand table backup could fail during executing copy job. You need to check results via looking in application logs. | ||
<br><br> | ||
On demand backup works for:<br> | ||
- non partitioned tables<br> | ||
- single partition of partitioned table (not specifying source partition id for partitioned table will not force performing backups) | ||
|
||
</p> | ||
|
||
<form name="restoreTableForm" onsubmit="return OnSubmitForm();"> | ||
<div class="form-group row"> | ||
<label for="sourceProjectId" class="col-sm-2 col-form-label">Source project id</label> | ||
<div class="col-sm-8"> | ||
<input type="text" class="form-control" id="sourceProjectId" name="projectId" | ||
placeholder="project id" required/> | ||
<small id="sourceProjectIdHelp" class="form-text text-muted">Source project id | ||
</small> | ||
</div> | ||
</div> | ||
<div class="form-group row"> | ||
<label for="sourceDatasetId" class="col-sm-2 col-form-label">Source dataset id</label> | ||
<div class="col-sm-8"> | ||
<input type="text" class="form-control" id="sourceDatasetId" name="datasetId" | ||
placeholder="dataset id"> | ||
<small id="sourceDatasetIdHelp" class="form-text text-muted">Source dataset id | ||
</small> | ||
</div> | ||
</div> | ||
<div class="form-group row"> | ||
<label for="sourceTableId" class="col-sm-2 col-form-label">Source table id</label> | ||
<div class="col-sm-8"> | ||
<input type="text" class="form-control" id="sourceTableId" name="tableId" | ||
placeholder="table id"> | ||
<small id="sourceTableIdHelp" class="form-text text-muted">Source table id </small> | ||
</div> | ||
</div> | ||
<div class="form-group row"> | ||
<label for="sourcePartitionId" class="col-sm-2 col-form-label">Source partition id</label> | ||
<div class="col-sm-8"> | ||
<input type="number" class="form-control" id="sourcePartitionId" name="partitionId" | ||
placeholder="partition id"> | ||
<small id="sourcePartitionIdHelp" class="form-text text-muted">Source partition id for partitioned table | ||
</small> | ||
</div> | ||
</div> | ||
</div> | ||
<div class="form-group row"> | ||
<div class="col-sm-4"> | ||
<button type="submit" class="btn btn-primary">Schedule backup</button> | ||
</div> | ||
</div> | ||
</form> | ||
|
||
<br><strong>Response:</strong> <br> | ||
<pre id="response"></pre> | ||
|
||
{% endblock %} |
Empty file.
Oops, something went wrong.