Skip to content

Commit

Permalink
Fixed issues #5155 and #5269. Created also missing test cases for aut…
Browse files Browse the repository at this point in the history 
…omatic backups
  • Loading branch information
@lvca
lvca committed Nov 4, 2015
1 parent 079abdb commit 3e0c438
Show file tree
Hide file tree
Showing 7 changed files with 316 additions and 49 deletions.
2 changes: 2 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/db/document/ODatabaseDocumentTx.java
View file
Expand Up @@ -2738,6 +2738,8 @@ public void restore(final InputStream in, final Map<String, Object> options, fin
storage = Orient.instance().loadStorage(url);

getStorage().restore(in, options, callable, iListener);

getMetadata().reload();
}

/**
Expand Down
9 changes: 8 additions & 1 deletion core/src/main/java/com/orientechnologies/orient/core/db/tool/ODatabaseImport.java
View file
Expand Up @@ -237,7 +237,7 @@ public Set convert(Set value) {
final Set result;

if (value instanceof OMVRBTreeRIDSet) {
OMVRBTreeRIDSet ridSet = new OMVRBTreeRIDSet(((OMVRBTreeRIDSet)value).getOwner());
OMVRBTreeRIDSet ridSet = new OMVRBTreeRIDSet(((OMVRBTreeRIDSet) value).getOwner());
ridSet.setAutoConvertToRecord(false);

result = ridSet;
Expand Down Expand Up @@ -369,6 +369,13 @@ public ODatabaseImport(final ODatabaseDocumentInternal database, final String iF
throws IOException {
super(database, iFileName, iListener);

if (iListener == null)
listener = new OCommandOutputListener() {
@Override
public void onMessage(String iText) {
}
};

InputStream inStream;
final BufferedInputStream bf = new BufferedInputStream(new FileInputStream(fileName));
bf.mark(1024);
Expand Down
17 changes: 12 additions & 5 deletions ...om/orientechnologies/orient/core/storage/impl/local/paginated/OLocalPaginatedStorage.java
View file
Expand Up @@ -20,7 +20,6 @@

package com.orientechnologies.orient.core.storage.impl.local.paginated;

import com.orientechnologies.common.concur.lock.OModificationOperationProhibitedException;
import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.io.OFileUtils;
import com.orientechnologies.common.io.OIOUtils;
Expand All @@ -45,11 +44,17 @@
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.ODiskWriteAheadLog;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber;
import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWriteAheadLog;
import sun.misc.IOUtils;

import java.io.*;
import java.lang.String;
import java.util.*;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -191,6 +196,8 @@ public void restore(InputStream in, Map<String, Object> options, final Callable<
close(true, false);

OZIPCompressionUtil.uncompressDirectory(in, getStoragePath(), iListener);

open(null, null, null);
}

@Override
Expand Down
4 changes: 4 additions & 0 deletions graphdb/config/orientdb-server-config.xml
View file
Expand Up @@ -48,6 +48,10 @@
<handler class="com.orientechnologies.orient.server.handler.OAutomaticBackup">
<parameters>
<parameter name="enabled" value="false"/>
<!-- CAN BE: FULL_BACKUP, INCREMENTAL_BACKUP, EXPORT -->
<parameter name="mode" value="FULL_BACKUP"/>
<!-- OPTION FOR EXPORT -->
<parameter name="exportOptions" value=""/>
<parameter name="delay" value="4h"/>
<parameter name="firstTime" value="23:00:00"/>
<parameter name="target.directory" value="backup"/>
Expand Down
159 changes: 116 additions & 43 deletions server/src/main/java/com/orientechnologies/orient/server/handler/OAutomaticBackup.java
View file
Expand Up @@ -30,6 +30,7 @@
import com.orientechnologies.orient.core.command.OCommandOutputListener;
import com.orientechnologies.orient.core.db.ODatabase;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.db.tool.ODatabaseExport;
import com.orientechnologies.orient.core.exception.OConfigurationException;
import com.orientechnologies.orient.core.metadata.security.OSecurityNull;
import com.orientechnologies.orient.server.OServer;
Expand All @@ -38,21 +39,40 @@

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TimerTask;

/**
* Automatically creates a backup at configured time. Starting from v2.2, this component is able also to create incremental backup
* and export of databases. If you need a mix of different modes, configure more instances of the same component.
*
* @author Luca Garulli
*/
public class OAutomaticBackup extends OServerPluginAbstract {

public enum VARIABLES {
DBNAME, DATE
}

private Date firstTime = null;
private long delay = -1;
private int bufferSize = 1048576;
private int compressionLevel = 9;
public enum MODE {
FULL_BACKUP, INCREMENTAL_BACKUP, EXPORT
}

private Date firstTime = null;
private long delay = -1;
private int bufferSize = 1048576;
private int compressionLevel = 9;
private MODE mode = MODE.FULL_BACKUP;
private String exportOptions;

private String targetDirectory = "backup";
private String targetFileName;
private Set<String> includeDatabases = new HashSet<String>();
Expand Down Expand Up @@ -80,8 +100,8 @@ else if (param.name.equalsIgnoreCase("firstTime")) {
firstTime = cal.getTime();
}
} catch (ParseException e) {
throw OException.wrapException(
new OConfigurationException("Parameter 'firstTime' has invalid format, expected: HH:mm:ss"), e);
throw OException
.wrapException(new OConfigurationException("Parameter 'firstTime' has invalid format, expected: HH:mm:ss"), e);
}
} else if (param.name.equalsIgnoreCase("target.directory"))
targetDirectory = param.value;
Expand All @@ -97,6 +117,10 @@ else if (param.name.equalsIgnoreCase("bufferSize"))
bufferSize = Integer.parseInt(param.value);
else if (param.name.equalsIgnoreCase("compressionLevel"))
compressionLevel = Integer.parseInt(param.value);
else if (param.name.equalsIgnoreCase("mode"))
mode = MODE.valueOf(param.value.toUpperCase());
else if (param.name.equalsIgnoreCase("exportOptions"))
exportOptions = param.value;
}

if (delay <= 0)
Expand All @@ -118,65 +142,64 @@ else if (param.name.equalsIgnoreCase("compressionLevel"))
final TimerTask timerTask = new TimerTask() {
@Override
public void run() {
OLogManager.instance().info(this, "[OAutomaticBackup] Scanning databases to backup...");
OLogManager.instance().info(this, "Scanning databases to backup...");

int ok = 0, errors = 0;

final Map<String, String> databaseNames = serverInstance.getAvailableStorageNames();
for (final Entry<String, String> dbName : databaseNames.entrySet()) {
final Map<String, String> databases = serverInstance.getAvailableStorageNames();
for (final Entry<String, String> database : databases.entrySet()) {
final String dbName = database.getKey();
final String dbURL = database.getValue();

boolean include;

if (includeDatabases.size() > 0)
include = includeDatabases.contains(dbName.getKey());
include = includeDatabases.contains(dbName);
else
include = true;

if (excludeDatabases.contains(dbName.getKey()))
if (excludeDatabases.contains(dbName))
include = false;

if (include) {
final String fileName = (String) OVariableParser.resolveVariables(targetFileName, OSystemVariableResolver.VAR_BEGIN,
OSystemVariableResolver.VAR_END, new OVariableParserListener() {
@Override
public String resolve(final String iVariable) {
if (iVariable.equalsIgnoreCase(VARIABLES.DBNAME.toString()))
return dbName.getKey();
else if (iVariable.startsWith(VARIABLES.DATE.toString())) {
return new SimpleDateFormat(iVariable.substring(VARIABLES.DATE.toString().length() + 1)).format(new Date());
}

// NOT FOUND
throw new IllegalArgumentException("Variable '" + iVariable + "' wasn't found");
}
});

final String exportFilePath = targetDirectory + fileName;
ODatabaseDocumentTx db = null;
try {

db = new ODatabaseDocumentTx(dbName.getValue());
db = new ODatabaseDocumentTx(dbURL);
db.setProperty(ODatabase.OPTIONS.SECURITY.toString(), OSecurityNull.class);
db.open("admin", "aaa");

final long begin = System.currentTimeMillis();

db.backup(new FileOutputStream(exportFilePath), null, null, new OCommandOutputListener() {
@Override
public void onMessage(String iText) {
OLogManager.instance().info(this, iText);
}
}, compressionLevel, bufferSize);

OLogManager.instance().info(
this,
"[OAutomaticBackup] - Backup of database '" + dbName.getValue() + "' completed in "
+ (System.currentTimeMillis() - begin) + "ms");
switch (mode) {
case FULL_BACKUP:
fullBackupDatabase(dbURL, targetDirectory + getFileName(database), db);

OLogManager.instance().info(this,
"Full Backup of database '" + dbURL + "' completed in " + (System.currentTimeMillis() - begin) + "ms");

break;

case INCREMENTAL_BACKUP:
incrementalBackupDatabase(dbURL, targetDirectory, db);

OLogManager.instance().info(this,
"Incremental Backup of database '" + dbURL + "' completed in " + (System.currentTimeMillis() - begin) + "ms");
break;

case EXPORT:
exportDatabase(dbURL, targetDirectory + getFileName(database), db);

OLogManager.instance().info(this,
"Export of database '" + dbURL + "' completed in " + (System.currentTimeMillis() - begin) + "ms");
break;
}

ok++;

} catch (Exception e) {

OLogManager.instance().error(this,
"[OAutomaticBackup] - Error on exporting database '" + dbName.getValue() + "' to file: " + exportFilePath, e);
OLogManager.instance().error(this, "Error on backup of database '" + dbURL + "' to directory: " + targetDirectory, e);
errors++;

} finally {
Expand All @@ -185,7 +208,7 @@ public void onMessage(String iText) {
}
}
}
OLogManager.instance().info(this, "[OAutomaticBackup] Backup finished: %d ok, %d errors", ok, errors);
OLogManager.instance().info(this, "Automatic Backup finished: %d ok, %d errors", ok, errors);
}
};

Expand All @@ -195,6 +218,56 @@ public void onMessage(String iText) {
Orient.instance().scheduleTask(timerTask, firstTime, delay);
}

protected void incrementalBackupDatabase(final String dbURL, String iPath, final ODatabaseDocumentTx db) throws IOException {
// APPEND DB NAME TO THE DIRECTORY NAME
if (!iPath.endsWith("/"))
iPath += "/";
iPath += db.getName();

db.incrementalBackup(iPath);
}

protected void fullBackupDatabase(final String dbURL, final String iPath, final ODatabaseDocumentTx db) throws IOException {
db.backup(new FileOutputStream(iPath), null, null, new OCommandOutputListener() {
@Override
public void onMessage(String iText) {
OLogManager.instance().info(this, iText);
}
}, compressionLevel, bufferSize);
}

protected void exportDatabase(final String dbURL, final String iPath, final ODatabaseDocumentTx db) throws IOException {

final ODatabaseExport exp = new ODatabaseExport(db, iPath, new OCommandOutputListener() {
@Override
public void onMessage(String iText) {
OLogManager.instance().info(this, iText);
}
});

if (exportOptions != null && !exportOptions.trim().isEmpty())
exp.setOptions(exportOptions.trim());

exp.exportDatabase().close();
}

protected String getFileName(final Entry<String, String> dbName) {
return (String) OVariableParser.resolveVariables(targetFileName, OSystemVariableResolver.VAR_BEGIN,
OSystemVariableResolver.VAR_END, new OVariableParserListener() {
@Override
public String resolve(final String iVariable) {
if (iVariable.equalsIgnoreCase(VARIABLES.DBNAME.toString()))
return dbName.getKey();
else if (iVariable.startsWith(VARIABLES.DATE.toString())) {
return new SimpleDateFormat(iVariable.substring(VARIABLES.DATE.toString().length() + 1)).format(new Date());
}

// NOT FOUND
throw new IllegalArgumentException("Variable '" + iVariable + "' was not found");
}
});
}

@Override
public String getName() {
return "automaticBackup";
Expand Down

0 comments on commit 3e0c438

Please sign in to comment.