Skip to content

Commit

Permalink
Merge branch 'develop' of github.com:eXist-db/exist into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@dizzzz
dizzzz committed Jun 14, 2014
2 parents 95e6458 + 4224651 commit 95d964d
Show file tree
Hide file tree
Showing 12 changed files with 280 additions and 224 deletions.
2 changes: 1 addition & 1 deletion extensions/fluent/src/org/exist/fluent/Database.java
View file
Expand Up @@ -190,7 +190,7 @@ public static boolean checkConsistency() {
try {
DBBroker broker = pool.enterServiceMode(pool.getSecurityManager().getSystemSubject());
try {
List<ErrorReport> errors = new ConsistencyCheck(broker, false).checkAll(NULL_PROGRESS_CALLBACK);
List<ErrorReport> errors = new ConsistencyCheck(broker, false, false).checkAll(NULL_PROGRESS_CALLBACK);
if (errors.isEmpty()) return true;
LOG.fatal("database corrupted");
for (ErrorReport error : errors) LOG.error(error.toString().replace("\n", " "));
Expand Down
5 changes: 5 additions & 0 deletions extensions/indexes/ngram/src/org/exist/indexing/ngram/NGramIndex.java
View file
Expand Up @@ -28,6 +28,7 @@
import org.exist.indexing.RawBackupSupport;
import org.exist.storage.BrokerPool;
import org.exist.storage.DBBroker;
import org.exist.storage.btree.BTree;
import org.exist.storage.btree.DBException;
import org.exist.storage.index.BFile;
import org.exist.util.DatabaseConfigurationException;
Expand Down Expand Up @@ -112,6 +113,10 @@ public int getN() {
return gramSize;
}

public BTree getStorage() {
return db;
}

public void backupToArchive(RawDataBackup backup) throws IOException {
OutputStream os = backup.newEntry(db.getFile().getName());
db.backupToStream(os);
Expand Down
120 changes: 96 additions & 24 deletions src/org/exist/backup/ConsistencyCheck.java
View file
Expand Up @@ -21,6 +21,7 @@
*/
package org.exist.backup;

import org.exist.storage.StorageAddress;
import org.w3c.dom.Node;

import org.exist.collections.Collection;
Expand All @@ -47,10 +48,7 @@

import java.io.IOException;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Stack;
import java.util.*;

import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
Expand All @@ -65,12 +63,19 @@ public class ConsistencyCheck
private DBBroker broker;
private int defaultIndexDepth;
private boolean directAccess = false;
private boolean checkDocs = false;

public ConsistencyCheck( DBBroker broker, boolean directAccess )
/**
* @param broker the db broker to use
* @param directAccess set to true to bypass the collections.dbx index and perform a low-level scan instead
* @param checkDocs set to true to perform additional checks on every document (slow)
*/
public ConsistencyCheck( DBBroker broker, boolean directAccess, boolean checkDocs)
{
this.broker = broker;
this.defaultIndexDepth = ( (NativeBroker)broker ).getDefaultIndexDepth();
this.directAccess = directAccess;
this.checkDocs = checkDocs;
}

/**
Expand Down Expand Up @@ -214,12 +219,47 @@ public void checkDocuments( ProgressCallback progress, List<ErrorReport> errorLi
try {
final DocumentCallback cb = new DocumentCallback( errorList, progress, true );
broker.getResourcesFailsafe( cb, directAccess );
cb.checkDocs();
}
finally {
AccountImpl.getSecurityProperties().enableCheckPasswords(true);
}
}

/**
* Check if data for the given XML document exists. Tries to load the document's root element.
* This check is certainly not as comprehensive as {@link #checkXMLTree(org.exist.dom.DocumentImpl)},
* but much faster.
*
* @param doc the document object to check
* @return
*/
public ErrorReport checkDocument(final DocumentImpl doc) {
final DOMFile domDb = ( (NativeBroker)broker ).getDOMFile();
return (ErrorReport)new DOMTransaction( this, domDb, Lock.WRITE_LOCK, doc ) {
public Object start() {
EmbeddedXMLStreamReader reader = null;
try {
final ElementImpl root = (ElementImpl)doc.getDocumentElement();
if (root == null) {
return new ErrorReport.ResourceError(ErrorReport.RESOURCE_ACCESS_FAILED, "Failed to access document data");
}
} catch( final Exception e ) {
e.printStackTrace();
return( new ErrorReport.ResourceError( org.exist.backup.ErrorReport.RESOURCE_ACCESS_FAILED, e.getMessage(), e ) );
} finally {
if (reader != null) {
try {
reader.close();
} catch (XMLStreamException e) {
e.printStackTrace();
}
}
}
return null;
}
}.run();
}

/**
* Check the persistent DOM of a document. The method traverses the entire node tree and checks it for consistency, including node relationships,
Expand All @@ -233,11 +273,11 @@ public ErrorReport checkXMLTree( final DocumentImpl doc )
{
final DOMFile domDb = ( (NativeBroker)broker ).getDOMFile();
return( (ErrorReport)new DOMTransaction( this, domDb, Lock.WRITE_LOCK, doc ) {
public Object start()
{
public Object start() {
EmbeddedXMLStreamReader reader = null;
try {
final ElementImpl root = (ElementImpl)doc.getDocumentElement();
final EmbeddedXMLStreamReader reader = broker.getXMLStreamReader( root, true );
reader = broker.getXMLStreamReader( root, true );
NodeId nodeId;
boolean attribsAllowed = false;
int expectedAttribs = 0;
Expand Down Expand Up @@ -348,6 +388,13 @@ public Object start()
}
finally {
elementStack.clear();
if (reader != null) {
try {
reader.close();
} catch (XMLStreamException e) {
e.printStackTrace();
}
}
}
}
}.run() );
Expand Down Expand Up @@ -385,6 +432,7 @@ private class DocumentCallback implements BTreeCallback
private boolean checkDocs;
private int lastPercentage = -1;
private Agent jmxAgent = AgentFactory.getInstance();
private ArrayList<DocumentImpl> docs = new ArrayList<>(8192);

private DocumentCallback( List<ErrorReport> errors, ProgressCallback progress, boolean checkDocs )
{
Expand Down Expand Up @@ -426,22 +474,8 @@ public boolean indexInfo( Value key, long pointer ) throws TerminatedException
}

if( ( type == DocumentImpl.XML_FILE ) && !directAccess ) {
final ErrorReport report = checkXMLTree( doc );

if( report != null ) {

if( report instanceof ErrorReport.ResourceError ) {
( (ErrorReport.ResourceError)report ).setDocumentId( docId );
}

if( errors != null ) {
errors.add( report );
}

if( progress != null ) {
progress.error( report );
}
}
// add to the list of pending documents. They will be checked later
docs.add(doc);
}
}
}
Expand All @@ -463,5 +497,43 @@ public boolean indexInfo( Value key, long pointer ) throws TerminatedException
}
return( true );
}

/**
* Sort the documents in the pending list by their storage page, then
* check each of them.
*/
public void checkDocs() {
DocumentImpl documents[] = new DocumentImpl[docs.size()];
docs.toArray(documents);
Arrays.sort(documents, new Comparator<DocumentImpl>() {
@Override
public int compare(DocumentImpl d1, DocumentImpl d2) {
final long a1 = StorageAddress.pageFromPointer(d1.getFirstChildAddress());
final long a2 = StorageAddress.pageFromPointer(d2.getFirstChildAddress());
return Long.compare(a1, a2);
}
});
for (DocumentImpl doc : documents) {
final ErrorReport report;
if (ConsistencyCheck.this.checkDocs) {
report = checkXMLTree(doc);
} else {
report = checkDocument(doc);
}
if( report != null ) {
if(report instanceof ErrorReport.ResourceError) {
( (ErrorReport.ResourceError)report ).setDocumentId( doc.getDocId() );
}

if(errors != null) {
errors.add(report);
}

if(progress != null) {
progress.error(report);
}
}
}
}
}
}
21 changes: 16 additions & 5 deletions src/org/exist/backup/ExportGUI.java
View file
Expand Up @@ -70,6 +70,7 @@ public class ExportGUI extends javax.swing.JFrame
private javax.swing.JCheckBox zipBtn;
private javax.swing.JCheckBox incrementalBtn;
private JCheckBox directAccessBtn;
private javax.swing.JCheckBox scanBtn;
private javax.swing.JLabel jLabel1;
private javax.swing.JLabel jLabel2;
private javax.swing.JMenu jMenu1;
Expand Down Expand Up @@ -156,7 +157,8 @@ private void initComponents()
jToolBar1 = new javax.swing.JToolBar();
startBtn = new javax.swing.JButton();
exportBtn = new javax.swing.JButton();
incrementalBtn = new JCheckBox( "Incremental backup" );
incrementalBtn = new JCheckBox( "Incremental" );
scanBtn = new JCheckBox( "Scan docs" );
directAccessBtn = new JCheckBox( "Direct access" );
zipBtn = new JCheckBox("Create ZIP");
outputDir = new javax.swing.JTextField();
Expand Down Expand Up @@ -246,6 +248,10 @@ public void actionPerformed( java.awt.event.ActionEvent evt )
jToolBar1.add( exportBtn );

jToolBar1.add( incrementalBtn );
scanBtn.setSelected(true);
scanBtn.setToolTipText( "Perform additional checks; scans every XML document" );
jToolBar1.add( scanBtn );
directAccessBtn.setToolTipText( "Bypass collection index by scanning collection store" );
jToolBar1.add( directAccessBtn );
jToolBar1.add( zipBtn );

Expand Down Expand Up @@ -496,12 +502,14 @@ public void error( String message, Throwable exception )

selected = zipBtn.getSelectedObjects();
final boolean zip = ( selected != null ) && ( selected[0] != null );

displayMessage( "Starting export ..." );
final long start = System.currentTimeMillis();
final SystemExport sysexport = new SystemExport( broker, callback, null, directAccess );
final File file = sysexport.export( exportTarget, incremental, zip, errorList );

displayMessage( "Export to " + file.getAbsolutePath() + " completed successfully." );
displayMessage( "Export took " + (System.currentTimeMillis() - start) + "ms.");
progress.setString( "" );
}
catch( final EXistException e ) {
Expand All @@ -524,9 +532,12 @@ private List<ErrorReport> checkDB()

try {
broker = pool.get( pool.getSecurityManager().getSystemSubject() );
final Object[] selected = directAccessBtn.getSelectedObjects();
final boolean directAccess = ( selected != null ) && ( selected[0] != null );
final ConsistencyCheck checker = new ConsistencyCheck( broker, directAccess );
Object[] selected = directAccessBtn.getSelectedObjects();
final boolean directAccess = ( selected != null ) && ( selected[0] != null );
selected = scanBtn.getSelectedObjects();
final boolean scan = ( selected != null ) && ( selected[0] != null );

final ConsistencyCheck checker = new ConsistencyCheck( broker, directAccess, scan );
final org.exist.backup.ConsistencyCheck.ProgressCallback cb = new ConsistencyCheck.ProgressCallback() {
public void startDocument( String path, int current, int count )
{
Expand Down
45 changes: 37 additions & 8 deletions src/org/exist/backup/ExportMain.java
View file
Expand Up @@ -50,6 +50,8 @@ public class ExportMain
private final static int NO_CHECK_OPT = 'n';
private final static int DIRECT_ACCESS_OPT = 'D';
private final static int ZIP_OPT = 'z';
private final static int CHECK_DOCS_OPT = 's';
private final static int VERBOSE_OPT = 'v';

private final static CLOptionDescriptor[] OPTIONS = new CLOptionDescriptor[] {
new CLOptionDescriptor( "help", CLOptionDescriptor.ARGUMENT_DISALLOWED, HELP_OPT, "print help on command line options and exit." ),
Expand All @@ -59,7 +61,11 @@ public class ExportMain
new CLOptionDescriptor( "export", CLOptionDescriptor.ARGUMENT_DISALLOWED, EXPORT_OPT, "export database contents while preserving as much data as possible" ),
new CLOptionDescriptor( "incremental", CLOptionDescriptor.ARGUMENT_DISALLOWED, INCREMENTAL_OPT, "create incremental backup (use with --export|-x)" ),
new CLOptionDescriptor( "nocheck", CLOptionDescriptor.ARGUMENT_DISALLOWED, NO_CHECK_OPT, "do not run a consistency check. Just export the data." ),
new CLOptionDescriptor( "zip", CLOptionDescriptor.ARGUMENT_DISALLOWED, ZIP_OPT, "write output to a ZIP instead of a file system directory" )
new CLOptionDescriptor( "check-docs", CLOptionDescriptor.ARGUMENT_DISALLOWED, CHECK_DOCS_OPT, "scan every document to find errors in the " +
"the nodes stored (costs time)" ),
new CLOptionDescriptor( "zip", CLOptionDescriptor.ARGUMENT_DISALLOWED, ZIP_OPT, "write output to a ZIP instead of a file system directory" ),
new CLOptionDescriptor( "verbose", CLOptionDescriptor.ARGUMENT_DISALLOWED, VERBOSE_OPT, "print processed resources " +
"to stdout" )
};

protected static BrokerPool startDB( String configFile )
Expand Down Expand Up @@ -100,6 +106,8 @@ public static void main( String[] args )
boolean direct = false;
boolean zip = false;
boolean nocheck = false;
boolean verbose = false;
boolean checkDocs = false;
String exportTarget = "export/";
String dbConfig = null;

Expand Down Expand Up @@ -145,10 +153,21 @@ public static void main( String[] args )
zip = true;
break;
}

case NO_CHECK_OPT: {
nocheck = true;
break;
}

case CHECK_DOCS_OPT: {
checkDocs = true;
break;
}

case VERBOSE_OPT: {
verbose = true;
break;
}
}
}

Expand All @@ -165,7 +184,7 @@ public static void main( String[] args )
List<ErrorReport> errors = null;

if(!nocheck) {
final ConsistencyCheck checker = new ConsistencyCheck( broker, direct );
final ConsistencyCheck checker = new ConsistencyCheck( broker, direct, checkDocs );
errors = checker.checkAll( new CheckCallback() );
}

Expand All @@ -182,8 +201,8 @@ public static void main( String[] args )
if( !dir.exists() ) {
dir.mkdirs();
}
final SystemExport sysexport = new SystemExport( broker, new Callback(), null, direct );
sysexport.export( exportTarget, incremental, true, errors );
final SystemExport sysexport = new SystemExport( broker, new Callback(verbose), null, direct );
sysexport.export( exportTarget, incremental, zip, errors );
}
}
catch( final EXistException e ) {
Expand All @@ -205,17 +224,27 @@ public static void main( String[] args )
System.exit( retval );
}

private static class Callback implements SystemExport.StatusCallback
{
private static class Callback implements SystemExport.StatusCallback {

private boolean verbose = false;

public Callback(boolean verbose) {
this.verbose = verbose;
}

public void startCollection( String path )
{
System.out.println( "Entering collection " + path + " ..." );
if (verbose) {
System.out.println("Entering collection " + path + " ...");
}
}


public void startDocument( String name, int count, int docsCount )
{
System.out.println( "Writing document " + name + " [" + count + " of " + docsCount + ']' );
if (verbose) {
System.out.println("Writing document " + name + " [" + (count + 1) + " of " + docsCount + ']');
}
}


Expand Down
3 changes: 3 additions & 0 deletions src/org/exist/collections/Collection.java
View file
Expand Up @@ -1006,6 +1006,9 @@ public interface InternalAccess {
*/
public void read(final DBBroker broker, final VariableByteInput istream) throws IOException, PermissionDeniedException {
collectionId = istream.readInt();
if (collectionId < 0) {
throw new PermissionDeniedException("Internal error reading collection: invalid collection id");
}
final int collLen = istream.readInt();
subCollections = new ObjectHashSet<XmldbURI>(collLen == 0 ? 19 : collLen); //TODO what is this number 19?
for (int i = 0; i < collLen; i++) {
Expand Down

0 comments on commit 95d964d

Please sign in to comment.