From 3d406fe16973bea5af6f51bde10f583546f3f6c3 Mon Sep 17 00:00:00 2001 From: Andy Seaborne Date: Fri, 4 Aug 2017 17:14:33 +0100 Subject: [PATCH 1/2] JENA-1379: Build NodeTables from ObjectFiles and BlockMgrs. Remove NodeTableBuilder Remove NodeTableTrans --- .../apache/jena/tdb/setup/BuilderStdDB.java | 32 -- .../apache/jena/tdb/setup/DatasetBuilder.java | 33 -- .../jena/tdb/setup/DatasetBuilderStd.java | 178 +++++--- .../jena/tdb/setup/NodeTableBuilder.java | 26 -- .../apache/jena/tdb/store/StorageConfig.java | 6 +- .../tdb/transaction/DatasetBuilderTxn.java | 134 ++---- .../jena/tdb/transaction/JournalControl.java | 54 +-- .../jena/tdb/transaction/NodeTableTrans.java | 413 ------------------ .../jena/tdb/transaction/ObjectFileTrans.java | 232 ++-------- .../transaction/ObjectFileTransComplex.java | 253 +++++++++++ .../jena/tdb/transaction/Transaction.java | 17 +- .../jena/tdb/extra/T_QuadsObjectIsNull.java | 110 ----- .../AbstractTestNodeTableTrans.java | 235 ---------- .../AbstractTestObjectFileTrans.java | 10 +- .../AbstractTestObjectFileTransComplex.java | 158 +++++++ .../tdb/transaction/TS_TransactionTDB.java | 7 +- ...ava => TestObjectFileTransComplexMem.java} | 14 +- ...=> TestObjectFileTransComplexStorage.java} | 42 +- 18 files changed, 641 insertions(+), 1313 deletions(-) delete mode 100644 jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilder.java delete mode 100644 jena-tdb/src/main/java/org/apache/jena/tdb/setup/NodeTableBuilder.java delete mode 100644 jena-tdb/src/main/java/org/apache/jena/tdb/transaction/NodeTableTrans.java create mode 100644 jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTransComplex.java delete mode 100644 jena-tdb/src/test/java/org/apache/jena/tdb/extra/T_QuadsObjectIsNull.java delete mode 100644 jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestNodeTableTrans.java create mode 100644 jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTransComplex.java rename jena-tdb/src/test/java/org/apache/jena/tdb/transaction/{TestNodeTableTransMem.java => TestObjectFileTransComplexMem.java} (76%) rename jena-tdb/src/test/java/org/apache/jena/tdb/transaction/{TestNodeTableTransDisk.java => TestObjectFileTransComplexStorage.java} (50%) diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/BuilderStdDB.java b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/BuilderStdDB.java index 2484b84c259..d7e59935b17 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/BuilderStdDB.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/BuilderStdDB.java @@ -22,18 +22,11 @@ import org.apache.jena.tdb.base.file.FileSet ; import org.apache.jena.tdb.base.objectfile.ObjectFile ; import org.apache.jena.tdb.base.record.RecordFactory ; -import org.apache.jena.tdb.index.Index ; -import org.apache.jena.tdb.index.IndexBuilder ; import org.apache.jena.tdb.index.RangeIndex ; import org.apache.jena.tdb.index.RangeIndexBuilder ; import org.apache.jena.tdb.lib.ColumnMap ; -import org.apache.jena.tdb.store.nodetable.NodeTable ; -import org.apache.jena.tdb.store.nodetable.NodeTableCache ; -import org.apache.jena.tdb.store.nodetable.NodeTableInline ; -import org.apache.jena.tdb.store.nodetable.NodeTableNative ; import org.apache.jena.tdb.store.tupletable.TupleIndex ; import org.apache.jena.tdb.store.tupletable.TupleIndexRecord ; -import org.apache.jena.tdb.sys.Names ; import org.apache.jena.tdb.sys.SystemTDB ; public class BuilderStdDB { @@ -56,31 +49,6 @@ public TupleIndex buildTupleIndex(FileSet fileSet, ColumnMap colMap, String name } } - public static class NodeTableBuilderStd implements NodeTableBuilder - { - private final IndexBuilder indexBuilder ; - private final ObjectFileBuilder objectFileBuilder ; - - public NodeTableBuilderStd(IndexBuilder indexBuilder, ObjectFileBuilder objectFileBuilder) { - this.indexBuilder = indexBuilder ; - this.objectFileBuilder = objectFileBuilder ; - } - - @Override - public NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) { - RecordFactory recordFactory = new RecordFactory(SystemTDB.LenNodeHash, SystemTDB.SizeOfNodeId) ; - Index idx = indexBuilder.buildIndex(fsIndex, recordFactory, params) ; - ObjectFile objectFile = objectFileBuilder.buildObjectFile(fsObjectFile, Names.extNodeData) ; - NodeTable nodeTable = new NodeTableNative(idx, objectFile) ; - nodeTable = NodeTableCache.create(nodeTable, - params.getNode2NodeIdCacheSize(), - params.getNodeId2NodeCacheSize(), - params.getNodeMissCacheSize()) ; - nodeTable = NodeTableInline.create(nodeTable) ; - return nodeTable ; - } - } - public static class ObjectFileBuilderStd implements ObjectFileBuilder { public ObjectFileBuilderStd() { } diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilder.java b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilder.java deleted file mode 100644 index 75a8fe99a87..00000000000 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilder.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.tdb.setup; - -import org.apache.jena.tdb.base.file.Location ; -import org.apache.jena.tdb.store.DatasetGraphTDB ; - -/** A builder of datasets in a Location (usually, a directory on disk) */ - -public interface DatasetBuilder -{ - /** - * Build a new Java object for a dataset. - * Location may already contains files for the state, or may need initializing. - */ - public DatasetGraphTDB build(Location location, StoreParams params) ; -} diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilderStd.java b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilderStd.java index d538c0a4acd..da82781ad08 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilderStd.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/DatasetBuilderStd.java @@ -35,17 +35,21 @@ import org.apache.jena.tdb.base.file.FileSet ; import org.apache.jena.tdb.base.file.Location ; import org.apache.jena.tdb.base.objectfile.ObjectFile; -import org.apache.jena.tdb.index.BuilderStdIndex ; -import org.apache.jena.tdb.index.IndexBuilder ; -import org.apache.jena.tdb.index.IndexParams ; -import org.apache.jena.tdb.index.RangeIndexBuilder ; +import org.apache.jena.tdb.base.record.RecordFactory ; +import org.apache.jena.tdb.index.* ; +import org.apache.jena.tdb.index.bplustree.BPlusTree ; +import org.apache.jena.tdb.index.bplustree.BPlusTreeParams ; import org.apache.jena.tdb.lib.ColumnMap ; import org.apache.jena.tdb.solver.OpExecutorTDB1 ; import org.apache.jena.tdb.store.* ; import org.apache.jena.tdb.store.nodetable.NodeTable ; +import org.apache.jena.tdb.store.nodetable.NodeTableCache ; +import org.apache.jena.tdb.store.nodetable.NodeTableInline ; +import org.apache.jena.tdb.store.nodetable.NodeTableNative ; import org.apache.jena.tdb.store.nodetupletable.NodeTupleTable ; import org.apache.jena.tdb.store.nodetupletable.NodeTupleTableConcrete ; import org.apache.jena.tdb.store.tupletable.TupleIndex ; +import org.apache.jena.tdb.store.tupletable.TupleIndexRecord ; import org.apache.jena.tdb.sys.* ; import org.slf4j.Logger ; @@ -54,15 +58,65 @@ * BlockMgr/BufferChannel/NodeTable for use by the transaction builder. */ -public class DatasetBuilderStd implements DatasetBuilder { +public class DatasetBuilderStd { private static final Logger log = TDB.logInfo ; - private NodeTableBuilder nodeTableBuilder ; - private TupleIndexBuilder tupleIndexBuilder ; - private Recorder recorder = null ; + private BlockMgrBuilder blockMgrBuilder = new BuilderStdIndex.BlockMgrBuilderStd(); + private ObjectFileBuilder objectFileBuilder = new BuilderStdDB.ObjectFileBuilderStd(); + + private void setupRecord() { + if ( this.blockMgrBuilder instanceof BlockMgrBuilderRecorder) + throw new TDBException("Already recording (BlockMgrBuilder)"); + if ( this.objectFileBuilder instanceof ObjectFileBuilderRecorder) + throw new TDBException("Already recording (ObjectFileBuilder)"); + + this.blockMgrBuilder = new BlockMgrBuilderRecorder(blockMgrBuilder, recorder); + this.objectFileBuilder= new ObjectFileBuilderRecorder(objectFileBuilder, recorder); + } + + private RangeIndex buildRangeIndex(FileSet fileSet, RecordFactory recordFactory, IndexParams indexParams) { + int blkSize = indexParams.getBlockSize() ; + int order = BPlusTreeParams.calcOrder(blkSize, recordFactory.recordLength()) ; + RangeIndex rIndex = createBPTree(fileSet, order, blockMgrBuilder, blockMgrBuilder, recordFactory, indexParams) ; + return rIndex ; + } + + private Index buildIndex(FileSet fileSet, RecordFactory recordFactory, IndexParams indexParams) { + // Cheap. + return buildRangeIndex(fileSet, recordFactory, indexParams) ; + } + + /** Knowing all the parameters, create a B+Tree */ + private RangeIndex createBPTree(FileSet fileset, int order, + BlockMgrBuilder blockMgrBuilderNodes, + BlockMgrBuilder blockMgrBuilderRecords, + RecordFactory factory, IndexParams indexParams) + { + // ---- Checking + { + int blockSize = indexParams.getBlockSize() ; + if (blockSize < 0 ) + throw new IllegalArgumentException("Negative blocksize: "+blockSize) ; + if (blockSize < 0 && order < 0) throw new IllegalArgumentException("Neither blocksize nor order specified") ; + if (blockSize >= 0 && order < 0) order = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; + if (blockSize >= 0 && order >= 0) + { + int order2 = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; + if (order != order2) + throw new IllegalArgumentException("Wrong order (" + order + "), calculated = " + order2) ; + } + } + + BPlusTreeParams params = new BPlusTreeParams(order, factory) ; + + BlockMgr blkMgrNodes = blockMgrBuilderNodes.buildBlockMgr(fileset, Names.bptExtTree, indexParams) ; + BlockMgr blkMgrRecords = blockMgrBuilderRecords.buildBlockMgr(fileset, Names.bptExtRecords, indexParams) ; + return BPlusTree.create(params, blkMgrNodes, blkMgrRecords) ; + } + + private Recorder recorder = new Recorder() ; /** - * * @param location * @return DatasetGraphTDB */ @@ -86,7 +140,6 @@ public static DatasetGraphTDB create(Location location, StoreParams appParams) { boolean newArea = TDBInternal.isNewDatabaseArea(location) ; StoreParams params = Build.decideStoreParams(location, newArea, appParams, locParams, dftParams) ; DatasetBuilderStd x = new DatasetBuilderStd() ; - x.standardSetup() ; DatasetGraphTDB dsg = x.build(location, params) ; return dsg ; } @@ -97,45 +150,29 @@ public static DatasetGraphTDB create(StoreParams params) { } public static DatasetBuilderStd stdBuilder() { - DatasetBuilderStd x = new DatasetBuilderStd() ; - x.standardSetup() ; - return x ; + return new DatasetBuilderStd() ; } - protected DatasetBuilderStd() { } - - // Used by DatasetBuilderTxn - public DatasetBuilderStd(BlockMgrBuilder blockMgrBuilder, NodeTableBuilder nodeTableBuilder) { - setupRecord(blockMgrBuilder, nodeTableBuilder) ; + protected DatasetBuilderStd() { + this(new BuilderStdIndex.BlockMgrBuilderStd(), + new BuilderStdDB.ObjectFileBuilderStd()); } - private void standardSetup() { - ObjectFileBuilder objectFileBuilder = new BuilderStdDB.ObjectFileBuilderStd() ; - BlockMgrBuilder blockMgrBuilder = new BuilderStdIndex.BlockMgrBuilderStd() ; - IndexBuilder indexBuilderNT = new BuilderStdIndex.IndexBuilderStd(blockMgrBuilder, blockMgrBuilder) ; - NodeTableBuilder nodeTableBuilder = new BuilderStdDB.NodeTableBuilderStd(indexBuilderNT, objectFileBuilder) ; - setupRecord(blockMgrBuilder, nodeTableBuilder) ; - } - - protected void setupRecord(BlockMgrBuilder blockMgrBuilder, NodeTableBuilder nodeTableBuilder) { - recorder = new Recorder() ; - BlockMgrBuilder blockMgrBuilderRec = new BlockMgrBuilderRecorder(blockMgrBuilder, recorder) ; - - IndexBuilder indexBuilder = new BuilderStdIndex.IndexBuilderStd(blockMgrBuilderRec, blockMgrBuilderRec) ; - RangeIndexBuilder rangeIndexBuilder = new BuilderStdIndex.RangeIndexBuilderStd(blockMgrBuilderRec, blockMgrBuilderRec) ; - - this.nodeTableBuilder = nodeTableBuilder ; - nodeTableBuilder = new NodeTableBuilderRecorder(nodeTableBuilder, recorder) ; - - TupleIndexBuilder tupleIndexBuilder = new BuilderStdDB.TupleIndexBuilderStd(rangeIndexBuilder) ; - set(nodeTableBuilder, tupleIndexBuilder) ; - } - - protected void set(NodeTableBuilder nodeTableBuilder, TupleIndexBuilder tupleIndexBuilder) { - this.nodeTableBuilder = nodeTableBuilder ; - this.tupleIndexBuilder = tupleIndexBuilder ; + public DatasetBuilderStd(BlockMgrBuilder blockMgrBuilder, ObjectFileBuilder objectFileBuilder) { + this.blockMgrBuilder = blockMgrBuilder; + this.objectFileBuilder = objectFileBuilder; + this.recorder = new Recorder() ; + // XXX YUK + setupRecord(); } +// private void standardSetup() { +// ObjectFileBuilder objectFileBuilder = new BuilderStdDB.ObjectFileBuilderStd() ; +// BlockMgrBuilder blockMgrBuilder = new BuilderStdIndex.BlockMgrBuilderStd() ; +// IndexBuilder indexBuilderNT = new BuilderStdIndex.IndexBuilderStd(blockMgrBuilder, blockMgrBuilder) ; +// NodeTableBuilder nodeTableBuilder = new BuilderStdDB.NodeTableBuilderStd(indexBuilderNT, objectFileBuilder) ; +// setupRecord(blockMgrBuilder, nodeTableBuilder) ; +// } private static void checkLocation(Location location) { if ( location.isMem() ) @@ -153,7 +190,6 @@ private static void checkLocation(Location location) { error(log, "Directory not writeable: "+dirname) ; } - @Override public DatasetGraphTDB build(Location location, StoreParams params) { // Ensure that there is global synchronization synchronized (DatasetBuilderStd.class) { @@ -184,7 +220,7 @@ private synchronized DatasetGraphTDB buildWorker(Location location, boolean writ ReorderTransformation transform = (_transform == null) ? chooseReorderTransformation(location) : _transform ; StorageConfig storageConfig = new StorageConfig(location, params, writeable, - recorder.blockMgrs, recorder.objectFiles, recorder.bufferChannels, recorder.nodeTables) ; + recorder.blockMgrs, recorder.objectFiles, recorder.bufferChannels) ; recorder.finish() ; @@ -287,24 +323,44 @@ protected TupleIndex makeTupleIndex(Location location, String name, String prima // Commonly, name == indexOrder. FileSet fs = new FileSet(location, name) ; ColumnMap colMap = new ColumnMap(primary, indexOrder) ; - return tupleIndexBuilder.buildTupleIndex(fs, colMap, indexOrder, params) ; + return /*tupleIndexBuilder.*/buildTupleIndex(fs, colMap, indexOrder, params) ; + } + + //------------- + private TupleIndex buildTupleIndex(FileSet fileSet, ColumnMap colMap, String name, StoreParams params) { + RecordFactory recordFactory = new RecordFactory(SystemTDB.SizeOfNodeId * colMap.length(), 0) ; + RangeIndex rIdx = /*rangeIndexBuilder.*/buildRangeIndex(fileSet, recordFactory, params) ; + TupleIndex tIdx = new TupleIndexRecord(colMap.length(), colMap, name, recordFactory, rIdx) ; + return tIdx ; } public NodeTable makeNodeTable(Location location, StoreParams params) { - FileSet fsNodeToId = new FileSet(location, params.getIndexNode2Id()) ; - FileSet fsId2Node = new FileSet(location, params.getIndexId2Node()) ; - NodeTable nt = nodeTableBuilder.buildNodeTable(fsNodeToId, fsId2Node, params) ; - return nt ; + return makeNodeTable$(location, params.getIndexNode2Id(), params.getIndexId2Node(), params); } /** Make a node table overriding the node->id and id->node table names */ private NodeTable makeNodeTable$(Location location, String indexNode2Id, String indexId2Node, StoreParams params) { FileSet fsNodeToId = new FileSet(location, indexNode2Id) ; FileSet fsId2Node = new FileSet(location, indexId2Node) ; - NodeTable nt = nodeTableBuilder.buildNodeTable(fsNodeToId, fsId2Node, params) ; + NodeTable nt = /*nodeTableBuilder.*/buildNodeTable(fsNodeToId, fsId2Node, params) ; return nt ; } + //------------- + + private NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) { + RecordFactory recordFactory = new RecordFactory(SystemTDB.LenNodeHash, SystemTDB.SizeOfNodeId) ; + Index idx = /*indexBuilder.*/buildIndex(fsIndex, recordFactory, params) ; + ObjectFile objectFile = objectFileBuilder.buildObjectFile(fsObjectFile, Names.extNodeData) ; + NodeTable nodeTable = new NodeTableNative(idx, objectFile) ; + nodeTable = NodeTableCache.create(nodeTable, + params.getNode2NodeIdCacheSize(), + params.getNodeId2NodeCacheSize(), + params.getNodeMissCacheSize()) ; + nodeTable = NodeTableInline.create(nodeTable) ; + return nodeTable ; + } + protected NodeTable makeNodeTableNoCache(Location location, String indexNode2Id, String indexId2Node, StoreParams params) { StoreParamsBuilder spb = StoreParams.builder(params) .node2NodeIdCacheSize(-1) @@ -386,26 +442,6 @@ interface RecordNodeTable { void record(FileRef fileRef, NodeTable nodeTable) ; } - static class NodeTableBuilderRecorder implements NodeTableBuilder { - private final NodeTableBuilder builder ; - private final RecordNodeTable recorder ; - - NodeTableBuilderRecorder(NodeTableBuilder ntb, RecordNodeTable recorder) { - this.builder = ntb ; - this.recorder = recorder ; - } - - @Override - public NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) { - NodeTable nt = builder.buildNodeTable(fsIndex, fsObjectFile, params) ; - // It just knows, right? - FileRef ref = FileRef.create(fsObjectFile.filename(Names.extNodeData)) ; - recorder.record(ref, nt) ; - return nt ; - } - - } - static class ObjectFileBuilderRecorder implements ObjectFileBuilder { private final ObjectFileBuilder builder ; private final RecordObjectFile recorder ; diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/NodeTableBuilder.java b/jena-tdb/src/main/java/org/apache/jena/tdb/setup/NodeTableBuilder.java deleted file mode 100644 index ebe90b3be9f..00000000000 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/setup/NodeTableBuilder.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.tdb.setup; - -import org.apache.jena.tdb.base.file.FileSet ; -import org.apache.jena.tdb.store.nodetable.NodeTable ; - -public interface NodeTableBuilder { - NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) ; -} diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/store/StorageConfig.java b/jena-tdb/src/main/java/org/apache/jena/tdb/store/StorageConfig.java index 1da389bd033..1ebbea2b695 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/store/StorageConfig.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/store/StorageConfig.java @@ -25,7 +25,6 @@ import org.apache.jena.tdb.base.file.Location ; import org.apache.jena.tdb.base.objectfile.ObjectFile; import org.apache.jena.tdb.setup.StoreParams ; -import org.apache.jena.tdb.store.nodetable.NodeTable; import org.apache.jena.tdb.sys.FileRef ; public class StorageConfig @@ -34,22 +33,19 @@ public class StorageConfig public final Map blockMgrs ; public final Map objectFiles; public final Map bufferChannels; - public final Map nodeTables; public final Location location ; public final boolean writeable ; public StorageConfig(Location location, StoreParams params, boolean writeable, Map blockMgrs, Map objectFiles, - Map bufferChannels, - Map nodeTables) + Map bufferChannels) { this.location = location ; this.params = params ; this.blockMgrs = blockMgrs ; this.objectFiles = objectFiles ; this.bufferChannels = bufferChannels ; - this.nodeTables = nodeTables; this.writeable = writeable ; } diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/DatasetBuilderTxn.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/DatasetBuilderTxn.java index e32bc2a66ed..49d72646b60 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/DatasetBuilderTxn.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/DatasetBuilderTxn.java @@ -23,30 +23,23 @@ import org.apache.jena.query.ReadWrite ; import org.apache.jena.tdb.TDBException ; import org.apache.jena.tdb.base.block.BlockMgr ; -import org.apache.jena.tdb.base.block.BlockMgrLogger ; import org.apache.jena.tdb.base.block.BlockMgrReadonly ; -import org.apache.jena.tdb.base.file.FileFactory; import org.apache.jena.tdb.base.file.FileSet ; import org.apache.jena.tdb.base.objectfile.ObjectFile ; -import org.apache.jena.tdb.base.objectfile.ObjectFileReadonly; -import org.apache.jena.tdb.base.record.RecordFactory; -import org.apache.jena.tdb.index.*; -import org.apache.jena.tdb.setup.*; +import org.apache.jena.tdb.base.objectfile.ObjectFileReadonly ; +import org.apache.jena.tdb.index.IndexParams ; +import org.apache.jena.tdb.setup.BlockMgrBuilder ; +import org.apache.jena.tdb.setup.DatasetBuilderStd ; +import org.apache.jena.tdb.setup.ObjectFileBuilder ; import org.apache.jena.tdb.store.DatasetGraphTDB ; -import org.apache.jena.tdb.store.nodetable.*; import org.apache.jena.tdb.sys.FileRef ; -import org.apache.jena.tdb.sys.Names; -import org.apache.jena.tdb.sys.SystemTDB; public class DatasetBuilderTxn { - // Ideally, don't make a DatasetGraphTDB to pass to new DatasetGraphTxn as it rips it apart. - // Context for the build. private final TransactionManager txnMgr ; private final Map blockMgrs ; private final Map objectFiles; - private final Map nodeTables; private final DatasetGraphTDB dsg ; private Transaction txn; @@ -54,7 +47,6 @@ public DatasetBuilderTxn(TransactionManager txnMgr, DatasetGraphTDB dsg) { this.txnMgr = txnMgr ; this.blockMgrs = dsg.getConfig().blockMgrs ; this.objectFiles = dsg.getConfig().objectFiles ; - this.nodeTables = dsg.getConfig().nodeTables ; this.dsg = dsg ; } @@ -77,72 +69,23 @@ DatasetGraphTxn build(Transaction txn, ReadWrite mode) { private DatasetGraphTDB buildReadonly() { BlockMgrBuilder blockMgrBuilder = new BlockMgrBuilderReadonly(); - NodeTableBuilder nodeTableBuilder = new NodeTableBuilderReadonly(); - DatasetBuilderStd x = new DatasetBuilderStd(blockMgrBuilder, nodeTableBuilder) ; + ObjectFileBuilder objectFileBuilder = new ObjectFileBuilderReadonly(); + DatasetBuilderStd x = new DatasetBuilderStd(blockMgrBuilder, objectFileBuilder) ; DatasetGraphTDB dsg2 = x._build(dsg.getLocation(), dsg.getConfig().params, false, dsg.getReorderTransform()) ; return dsg2 ; } private DatasetGraphTDB buildWritable() { BlockMgrBuilder blockMgrBuilder = new BlockMgrBuilderTx() ; - NodeTableBuilder nodeTableBuilder = new NodeTableBuilderTx(); - DatasetBuilderStd x = new DatasetBuilderStd(blockMgrBuilder, nodeTableBuilder) ; + ObjectFileBuilder objectFileBuilder = new ObjectFileBuilderTx(); + DatasetBuilderStd x = new DatasetBuilderStd(blockMgrBuilder, objectFileBuilder); DatasetGraphTDB dsg2 = x._build(dsg.getLocation(), dsg.getConfig().params, true, dsg.getReorderTransform()) ; dsg2.getContext().putAll(dsg.getContext()) ; return dsg2 ; } - // ---- Add logging to a BlockMgr when built. - static BlockMgrBuilder logging(BlockMgrBuilder other) { return new BlockMgrBuilderLogger(other) ; } - - static class BlockMgrBuilderLogger implements BlockMgrBuilder { - public BlockMgrBuilder other ; - - public BlockMgrBuilderLogger(BlockMgrBuilder other) { - this.other = other ; - } - - @Override - public BlockMgr buildBlockMgr(FileSet fileSet, String ext, IndexParams params) { - BlockMgr blkMgr = other.buildBlockMgr(fileSet, ext, params) ; - blkMgr = new BlockMgrLogger(blkMgr.getLabel(), blkMgr, true) ; - return blkMgr ; - } - } - // ---- Build transactional versions for update. - class NodeTableBuilderTx implements NodeTableBuilder - { - @Override - public NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) { - FileRef ref = FileRef.create(fsObjectFile.filename(Names.extNodeData)) ; - NodeTable ntBase = nodeTables.get(ref) ; - if ( ntBase == null ) - throw new TDBException("No NodeTable for " + ref) ; - - RecordFactory recordFactory = new RecordFactory(SystemTDB.LenNodeHash, SystemTDB.SizeOfNodeId) ; - Index idx = new IndexMap(recordFactory) ; - String objFilename = fsObjectFile.filename(Names.extNodeData + "-" + Names.extJournal) ; - ObjectFile objectFile ; - - if ( fsObjectFile.isMem() ) - objectFile = FileFactory.createObjectFileMem(objFilename) ; - else - objectFile = FileFactory.createObjectFileDisk(objFilename) ; - - // Allow for a modified base NodeTable. - NodeTable ntBaseTrans = ntBase; - - NodeTableTrans ntt = new NodeTableTrans(txn, fsObjectFile.getBasename(), ntBaseTrans, idx, objectFile) ; - txn.addComponent(ntt) ; - - // Add inline wrapper. - NodeTable nt = NodeTableInline.create(ntt) ; - return nt ; - } - } - class BlockMgrBuilderTx implements BlockMgrBuilder { @Override @@ -150,33 +93,32 @@ public BlockMgr buildBlockMgr(FileSet fileSet, String ext, IndexParams params) { // Find from file ref. FileRef ref = FileRef.create(fileSet, ext) ; BlockMgr baseMgr = blockMgrs.get(ref) ; - if ( baseMgr == null ) - throw new TDBException("No BlockMgr for " + ref) ; + if ( baseMgr == null ) { + //System.out.flush(); + System.out.println("No BlockMgr for " + ref+" : "+blockMgrs.keySet()); + //throw new TDBException("No BlockMgr for " + ref) ; + } BlockMgrJournal blkMgr = new BlockMgrJournal(txn, ref, baseMgr) ; txn.addComponent(blkMgr) ; return blkMgr ; } } - // Object files currently don't need journalling. Because they are append only, they - // are "self journalling" - we can append to them while used read-only elsewhere if - // the index to access them is transactional and does not point into the appended new - // bytes. -// class ObjectFileBuilderTx implements ObjectFileBuilder -// { -// @Override -// public ObjectFile buildObjectFile(FileSet fileSet, String ext) { -// FileRef ref = FileRef.create(fileSet, ext) ; -// ObjectFile baseObjFile = objectFiles.get(ref); -// if ( baseObjFile == null ) -// throw new TDBException("No ObjectFile for " + ref) ; -// ObjectFileJournal objFile = new ObjectFileJournal(txn, ref, baseObjectFile); -// return objFile; -// } -// } - + class ObjectFileBuilderTx implements ObjectFileBuilder + { + @Override + public ObjectFile buildObjectFile(FileSet fileSet, String ext) { + FileRef ref = FileRef.create(fileSet, ext) ; + ObjectFile base = objectFiles.get(ref) ; + // Just write to the (append only) ObjectFile and manage aborts. + ObjectFileTrans objFileTxn = new ObjectFileTrans(txn, base) ; + txn.addComponent(objFileTxn); + return objFileTxn; + } + } + // ---- Build passthrough versions for readonly access - + class BlockMgrBuilderReadonly implements BlockMgrBuilder { @Override @@ -190,28 +132,14 @@ public BlockMgr buildBlockMgr(FileSet fileSet, String ext, IndexParams params) { } } - class NodeTableBuilderReadonly implements NodeTableBuilder - { - @Override - public NodeTable buildNodeTable(FileSet fsIndex, FileSet fsObjectFile, StoreParams params) { - FileRef ref = FileRef.create(fsObjectFile.filename(Names.extNodeData)) ; - NodeTable nt = nodeTables.get(ref) ; - nt = new NodeTableReadonly(nt) ; - return nt ; - } - } - class ObjectFileBuilderReadonly implements ObjectFileBuilder { @Override public ObjectFile buildObjectFile(FileSet fileSet, String ext) { FileRef ref = FileRef.create(fileSet, ext) ; - ObjectFile objFile = objectFiles.get(ref); - if ( objFile == null ) - throw new TDBException("No ObjectFile for " + ref) ; - objFile = new ObjectFileReadonly(objFile) ; - return objFile; + ObjectFile file = objectFiles.get(ref) ; + return new ObjectFileReadonly(file); } } - } + diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java index f384908e3d2..03a5d32ef52 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java @@ -29,23 +29,16 @@ import org.apache.jena.atlas.iterator.Iter ; import org.apache.jena.atlas.lib.FileOps ; -import org.apache.jena.atlas.logging.FmtLog; import org.apache.jena.tdb.TDBException ; import org.apache.jena.tdb.base.block.Block ; import org.apache.jena.tdb.base.block.BlockMgr ; import org.apache.jena.tdb.base.file.BufferChannel ; import org.apache.jena.tdb.base.file.BufferChannelFile ; -import org.apache.jena.tdb.base.file.FileFactory; import org.apache.jena.tdb.base.file.Location ; -import org.apache.jena.tdb.base.objectfile.ObjectFile; -import org.apache.jena.tdb.base.record.RecordFactory; -import org.apache.jena.tdb.index.IndexMap; import org.apache.jena.tdb.store.DatasetGraphTDB ; import org.apache.jena.tdb.store.StorageConfig ; -import org.apache.jena.tdb.store.nodetable.NodeTable; import org.apache.jena.tdb.sys.FileRef ; import org.apache.jena.tdb.sys.Names ; -import org.apache.jena.tdb.sys.SystemTDB; import org.slf4j.Logger ; import org.slf4j.LoggerFactory ; @@ -92,17 +85,13 @@ public static void recovery(DatasetGraphTDB dsg) if ( journal == null || journal.isEmpty() ) return ; - - for ( FileRef fileRef : dsg.getConfig().nodeTables.keySet() ) + for ( FileRef fileRef : dsg.getConfig().objectFiles.keySet() ) recoverNodeDat(dsg, fileRef) ; - // Not used currently. -// for ( FileRef fileRef : dsg.getConfig().objectFiles.keySet() ) -// recoverObjectFile(dsg, fileRef) ; recoverFromJournal(dsg.getConfig(), journal) ; journal.close() ; // Recovery complete. Tidy up. Node journal files have already been handled. - if ( journal.getFilename() != null ) + if ( journal.getFilename() != null ) { if ( FileOps.exists(journal.getFilename()) ) FileOps.delete(journal.getFilename()) ; @@ -201,35 +190,24 @@ private static void recoverSegment(Journal jrnl, long startPosn, long endPosn, S } finally { Iter.close(iter) ; } } - private static void recoverObjectFile(DatasetGraphTDB dsg, FileRef fileRef) { - FmtLog.warn(log, "Not recovered: recoverObjectFile[%s]", fileRef); - } - /** Recover a node data file (".dat"). - * Node data files are append-only so recovering, then not using the data is safe. - * Node data file is a precursor for full recovery that works from the master journal. + * Node data files are append-only so recovering. + * This code is only for ObjectFileTransComplex. */ private static void recoverNodeDat(DatasetGraphTDB dsg, FileRef fileRef) { - // See DatasetBuilderTxn - same name generation code. - - RecordFactory recordFactory = new RecordFactory(SystemTDB.LenNodeHash, SystemTDB.SizeOfNodeId) ; - NodeTable baseNodeTable = dsg.getConfig().nodeTables.get(fileRef) ; - String objFilename = fileRef.getFilename()+"-"+Names.extJournal ; - objFilename = dsg.getLocation().absolute(objFilename) ; - File jrnlFile = new File(objFilename) ; - if ( jrnlFile.exists() && jrnlFile.length() > 0 ) - { - syslog.info("Recovering node data: "+fileRef.getFilename()) ; - ObjectFile dataJrnl = FileFactory.createObjectFileDisk(objFilename) ; - NodeTableTrans ntt = new NodeTableTrans(null, objFilename, baseNodeTable, new IndexMap(recordFactory), dataJrnl) ; - ntt.append() ; - ntt.close() ; - dataJrnl.close() ; - baseNodeTable.sync() ; - } - if ( jrnlFile.exists() ) - FileOps.delete(objFilename) ; +// // See DatasetBuilderTxn - same name generation code. +// String objFilename = fileRef.getFilename()+"-"+Names.extJournal ; +// objFilename = dsg.getLocation().absolute(objFilename) ; +// File jrnlFile = new File(objFilename) ; +// if ( jrnlFile.exists() && jrnlFile.length() > 0 ) +// { +// syslog.info("Clearing node data: "+fileRef.getFilename()) ; +// ObjectFile dataJrnl = FileFactory.createObjectFileDisk(objFilename) ; +// dataJrnl.truncate(0); +// } +// if ( jrnlFile.exists() ) +// FileOps.delete(objFilename) ; } public static void replay(Transaction transaction) diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/NodeTableTrans.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/NodeTableTrans.java deleted file mode 100644 index 98615e10888..00000000000 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/NodeTableTrans.java +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.tdb.transaction; - -import static org.apache.jena.atlas.logging.FmtLog.warn ; - -import java.nio.ByteBuffer ; -import java.util.Iterator ; - -import org.apache.jena.atlas.iterator.Iter ; -import org.apache.jena.atlas.lib.ByteBufferLib ; -import org.apache.jena.atlas.lib.Pair ; -import org.apache.jena.graph.Node ; -import org.apache.jena.tdb.TDBException ; -import org.apache.jena.tdb.base.objectfile.ObjectFile ; -import org.apache.jena.tdb.base.record.Record ; -import org.apache.jena.tdb.index.Index ; -import org.apache.jena.tdb.store.NodeId ; -import org.apache.jena.tdb.store.nodetable.NodeTable ; -import org.apache.jena.tdb.store.nodetable.NodeTableCache ; -import org.apache.jena.tdb.store.nodetable.NodeTableInline ; -import org.apache.jena.tdb.store.nodetable.NodeTableNative ; -import org.slf4j.Logger ; -import org.slf4j.LoggerFactory ; - -public class NodeTableTrans implements NodeTable, TransactionLifecycle -{ - private static Logger log = LoggerFactory.getLogger(NodeTableTrans.class) ; - // TODO flag to note is any work is needed on commit. - private final NodeTable base ; - private long allocOffset ; - - private NodeTable nodeTableJournal = null ; - private static int CacheSize = 10000 ; // Make configurable - private boolean passthrough = false ; - - private Index nodeIndex ; - private ObjectFile journalObjFile ; - // Start of the journal file for this transaction. - // Always zero currently but allows for future - private long journalObjFileStartOffset ; - private final String label ; - private final Transaction txn ; // Can be null (during recovery). - - public NodeTableTrans(Transaction txn, String label, NodeTable sub, Index nodeIndex, ObjectFile journalObjFile) - { - this.txn = txn ; - this.base = sub ; - this.nodeIndex = nodeIndex ; - // Workspace for - this.journalObjFile = journalObjFile ; - // Clear bytes from an old run - // (a crash while writing means the old transaction did not commit - // any bytes in the file are junk) - // This is coupled to the fact the prepare phase does the actually data writing. - journalObjFile.truncate(0) ; - this.label = label ; - } - - public void setPassthrough(boolean v) { passthrough = v ; } - public NodeTable getBaseNodeTable() { return base ; } - public NodeTable getJournalTable() { return nodeTableJournal ; } - public Transaction getTransaction() { return txn ; } - - @Override - public NodeId getAllocateNodeId(Node node) - { - if ( passthrough ) return base.getAllocateNodeId(node) ; - NodeId nodeId = getNodeIdForNode(node) ; - if ( ! NodeId.isDoesNotExist(nodeId) ) - return nodeId ; - // add to journal - nodeId = allocate(node) ; - return nodeId ; - } - - @Override - public NodeId getNodeIdForNode(Node node) - { - if ( node == Node.ANY ) - return NodeId.NodeIdAny ; - if ( passthrough ) return base.getNodeIdForNode(node) ; - NodeId nodeId = nodeTableJournal.getNodeIdForNode(node) ; - if ( ! NodeId.isDoesNotExist(nodeId) ) - return mapFromJournal(nodeId) ; - nodeId = base.getNodeIdForNode(node) ; - return nodeId ; - } - - @Override - public Node getNodeForNodeId(NodeId id) - { - if ( NodeId.isAny(id) ) - return Node.ANY ; - if ( passthrough ) return base.getNodeForNodeId(id) ; - long x = id.getId() ; - if ( x < allocOffset ) - return base.getNodeForNodeId(id) ; - id = mapToJournal(id) ; - Node node = nodeTableJournal.getNodeForNodeId(id) ; - return node ; - } - - @Override - public boolean containsNode(Node node) { - NodeId x = getNodeIdForNode(node) ; - return NodeId.isDoesNotExist(x) ; - } - - @Override - public boolean containsNodeId(NodeId nodeId) { - Node x = getNodeForNodeId(nodeId) ; - return x == null ; - } - - - - /** Convert from a id to the id in the "journal" file */ - private NodeId mapToJournal(NodeId id) - { - if ( passthrough ) - throw new TDBTransactionException("Not in an active transaction") ; - if ( NodeId.isInline(id) ) - return id ; - return NodeId.create(id.getId()-allocOffset) ; - } - - /** Convert from a id in other to an external id */ - private NodeId mapFromJournal(NodeId id) - { - if ( passthrough ) - throw new TDBTransactionException("Not in an active transaction") ; - if ( NodeId.isInline(id) ) - return id ; - return NodeId.create(id.getId()+allocOffset) ; - } - - private NodeId allocate(Node node) - { - NodeId nodeId = nodeTableJournal.getAllocateNodeId(node) ; - nodeId = mapFromJournal(nodeId) ; - return nodeId ; - } - - @Override - public NodeId allocOffset() - { - if ( passthrough ) return base.allocOffset() ; - // If we have done the append stage, this is invalid as the base may change under our feet - // Would need to track base operations. - NodeId x1 = nodeTableJournal.allocOffset() ; - NodeId x2 = mapFromJournal(x1) ; - return x2 ; - } - - @Override - public void begin(Transaction txn) - { - //debug("%s begin", txn.getLabel()) ; - - if ( this.txn.getTxnId() != txn.getTxnId() ) - throw new TDBException(String.format("Different transactions: %s %s", this.txn.getLabel(), txn.getLabel())) ; - if ( passthrough ) - throw new TDBException("Already active") ; - passthrough = false ; - - allocOffset = base.allocOffset().getId() ; - // base node table empty e.g. first use. - journalObjFileStartOffset = journalObjFile.length() ; - // Because the data is written in prepare, the journal of object data is - // always empty at the start of a transaction. - if ( journalObjFileStartOffset != 0 ) - warn(log, "%s journalStartOffset not zero: %d/0x%02X",txn.getLabel(), journalObjFileStartOffset, journalObjFileStartOffset) ; - allocOffset += journalObjFileStartOffset ; - - this.nodeTableJournal = new NodeTableNative(nodeIndex, journalObjFile) ; - this.nodeTableJournal = NodeTableCache.create(nodeTableJournal, CacheSize, CacheSize, 100) ; - // This class knows about non-mappable inline values. mapToJournal(NodeId)/mapFromJournal. - this.nodeTableJournal = NodeTableInline.create(nodeTableJournal) ; - } - - static public boolean APPEND_LOG = false ; - - /** Copy from the journal file to the real file */ - /*package*/ void append() - { - Iterator> iter = nodeTableJournal.all() ; - Pair firstPair = null ; - Pair lastPair = null ; - - for ( ; iter.hasNext() ; ) - { - Pair x = iter.next() ; - - if ( firstPair == null ) - firstPair = x ; - lastPair = x ; - - NodeId nodeId = x.getLeft() ; - Node node = x.getRight() ; - debug(" append: %s -> %s", x, mapFromJournal(nodeId)) ; - // This does the write. - NodeId nodeId2 = base.getAllocateNodeId(node) ; - if ( ! nodeId2.equals(mapFromJournal(nodeId)) ) - inconsistent(node, nodeId, nodeId2) ; - } - } - - private void inconsistent(Node node , NodeId nodeId , NodeId nodeId2 ) - { - String msg = String.format("Different ids for %s: allocated: expected %s, got %s", node, mapFromJournal(nodeId), nodeId2) ; - System.err.println() ; - System.err.println() ; - System.err.println(msg) ; - dump() ; - System.err.println() ; - throw new TDBException(msg) ; - } - - // Debugging only - private void dump() - { - System.err.println(">>>>>>>>>>") ; - System.err.println("label = "+label) ; - System.err.println("txn = "+txn) ; - System.err.println("offset = "+allocOffset) ; - System.err.println("journalStartOffset = "+journalObjFileStartOffset) ; - System.err.println("journal = "+journalObjFile.getLabel()) ; - if ( true ) - return ; - - System.err.println("nodeTableJournal >>>") ; - Iterator> iter = nodeTableJournal.all() ; - for ( ; iter.hasNext() ; ) - { - Pair x = iter.next() ; - NodeId nodeId = x.getLeft() ; - Node node = x.getRight() ; - NodeId mapped = mapFromJournal(nodeId) ; - //debug("append: %s -> %s", x, mapFromJournal(nodeId)) ; - // This does the write. - NodeId nodeId2 = base.getAllocateNodeId(node) ; - System.err.println(x + " mapped=" + mapped + " getAlloc="+nodeId2) ; - } - - System.err.println("journal >>>") ; - Iterator> iter1 = this.journalObjFile.all() ; - for ( ; iter1.hasNext() ; ) - { - Pair p = iter1.next() ; - System.err.println(p.getLeft()+" : "+p.getRight()) ; - ByteBufferLib.print(System.err, p.getRight()) ; - } - - System.err.println("nodeIndex >>>") ; - Iterator iter2 = this.nodeIndex.iterator() ; - for ( ; iter2.hasNext() ; ) - { - Record r = iter2.next() ; - System.err.println(r) ; - } - System.err.println("<<<<<<<<<<") ; - } - - @Override - public void commitPrepare(Transaction txn) - { - // The index "node2id", which is Node hash to NodeId, is done because it has a BlockMgr. - debug("commitPrepare") ; - - // The node table is append-only so it can be written during prepare. - // The index isn't written (via the transaction journal) until enact. - if ( nodeTableJournal == null ) - throw new TDBTransactionException(txn.getLabel()+": Not in a transaction for a commit to happen") ; - writeNodeJournal() ; - - if ( journalObjFile != null && journalObjFile.length() != 0 ) - { - long x = journalObjFile.length() ; - throw new TDBTransactionException(txn.getLabel()+": journalObjFile not cleared ("+x+")") ; - } - } - - @Override - public void commitEnact(Transaction txn) - { - debug("commitEnact") ; - // The work was done in commitPrepare, using the fact that node data file - // is append only. Until here, pointers to the extra data aren't available - // until the index is written. - // The index is written via the transaction journal. - //writeJournal() ; - } - - private void writeNodeJournal() - { - long expected = base.allocOffset().getId() ; - long len = journalObjFile.length() ; - if ( expected != allocOffset ) - warn(log, "Inconsistency: base.allocOffset() = %d : allocOffset = %d", expected, allocOffset) ; - - long newbase = -1 ; - // Copy to the base NodeTable. - append() ; - // Reset (in case we use this again) - nodeIndex.clear() ; - journalObjFile.truncate(journalObjFileStartOffset) ; // Side effect is a buffer flush. - //journalObjFile.sync() ; - journalObjFile.close() ; // Side effect is a buffer flush. - journalObjFile = null ; - base.sync() ; - allocOffset = -99 ; // base.allocOffset().getId() ; // Will be invalid as we may write through to the base table later. - passthrough = true ; - } - - @Override - public void commitClearup(Transaction txn) - { - debug("commitClearup") ; - finish() ; - } - - @Override - public void abort(Transaction txn) - { - debug("abort") ; - if ( nodeTableJournal == null ) - throw new TDBTransactionException(txn.getLabel()+": Not in a transaction for a commit to happen") ; - // Ensure the cache does not flush. - nodeTableJournal = null ; - // then make sure the journal file is empty. - if ( journalObjFile != null ) - { - journalObjFile.truncate(journalObjFileStartOffset) ; - journalObjFile.sync() ; - } - finish() ; - } - - private void finish() - { - close() ; - passthrough = true ; - nodeTableJournal = null ; - journalObjFile = null ; - } - - @Override - public Iterator> all() - { - // Better would be to convert the spill file format. - return Iter.concat(base.all(), nodeTableJournal.all()) ; - } - - @Override - public boolean isEmpty() - { - return nodeTableJournal.isEmpty() && base.isEmpty() ; - } - - @Override - public void sync() - { - if ( passthrough ) - base.sync() ; - } - - @Override - public void close() - { - if ( nodeIndex != null ) - nodeIndex.close() ; - nodeIndex = null ; - // Closing the journal flushes it; i.e. disk IO. - if ( journalObjFile != null ) - journalObjFile.close() ; - journalObjFile = null ; - } - - @Override - public String toString() { return "NodeTableTrans:"+label+"(#"+Integer.toHexString(super.hashCode())+")" ; } - - private void debug(String fmt, Object... args) - { - if ( log.isDebugEnabled() ) - { - String x = String.format(fmt, args) ; - log.debug(label+": "+x) ; - } - } - - // Return the base table, not the nodeTableJournal - @Override - public NodeTable wrapped() { - return base ; - } -} diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTrans.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTrans.java index 82059ffe43d..d88ea1e2a4b 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTrans.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTrans.java @@ -18,223 +18,47 @@ package org.apache.jena.tdb.transaction; -import java.nio.ByteBuffer ; -import java.util.Iterator ; - -import org.apache.jena.atlas.iterator.Iter ; -import org.apache.jena.atlas.lib.Pair ; -import org.apache.jena.atlas.lib.StrUtils ; -import org.apache.jena.atlas.logging.Log ; -import org.apache.jena.tdb.base.block.Block ; -import org.apache.jena.tdb.base.file.FileException ; import org.apache.jena.tdb.base.objectfile.ObjectFile ; - -public class ObjectFileTrans implements ObjectFile, TransactionLifecycle -{ - private final ObjectFile transObjects ; - private long otherAllocOffset ; // record where we start allocating - private boolean passthrough = false ; - private boolean inTransaction = false ; - private final ObjectFile base ; - - // For recovery replay, we need to truncate "base" first. - - public ObjectFileTrans(Transaction txn, ObjectFile base, ObjectFile other) - { - // The "other" object file must use the same allocation policy. - this.base = base ; - this.transObjects = other ; - inTransaction = false ; - - // [TxTDB:PATCH-UP] Begin is not being called. - this.otherAllocOffset = base.length() ; - //Log.info(this, getLabel()+": otherAllocOffset = "+otherAllocOffset) ; - } - - // Begin read ==> passthrough. - - @Override - public void begin(Transaction txn) - { - passthrough = false ; - inTransaction = true ; - transObjects.reposition(0) ; - this.otherAllocOffset = base.length() ; - } - - @Override - public void commitPrepare(Transaction txn) - { - if ( ! inTransaction ) - throw new TDBTransactionException("Not in a transaction for a commit to happen") ; - transObjects.sync() ; - } - - @Override - public void commitEnact(Transaction txn) - { - if ( ! inTransaction ) - throw new TDBTransactionException("Not in a transaction for a commit to happen") ; - append() ; - base.sync() ; - transObjects.reposition(0) ; - } - - @Override - public void abort(Transaction txn) - { - transObjects.reposition(0) ; - } - - @Override - public void commitClearup(Transaction txn) - { - transObjects.truncate(0) ; - passthrough = true ; - } - - /** Copy from the temporary file to the real file */ - private void append() - { - // We could write directly to the real file if: - // we record the truncate point needed for an abort - // manage partial final writes - // deny the existence of nodes after the transaction mark. - // Later - stay simple for now. - - // Truncate/position the ObjectFile. - base.reposition(otherAllocOffset) ; - - Iterator> iter = transObjects.all() ; - for ( ; iter.hasNext() ; ) - { - Pair p = iter.next() ; - String s = StrUtils.fromUTF8bytes(p.getRight().array()) ; - - long x = base.write(p.getRight()) ; - - if ( p.getLeft()+otherAllocOffset != x ) - throw new FileException("Expected id of "+(p.getLeft()+otherAllocOffset)+", got an id of "+x) ; - } - } - - //public void setPassthrough(boolean v) { passthrough = v ; } - - @Override - public void reposition(long id) - { - if ( passthrough ) { base.reposition(id) ; return ; } - if ( id > otherAllocOffset ) - { - transObjects.reposition(mapToOther(id)) ; - return ; - } - - Log.warn(this, "Unexpected: Attempt to reposition over base file") ; - base.reposition(id) ; - transObjects.reposition(0) ; - otherAllocOffset = base.length() ; - } - - @Override - public void truncate(long id) - { - if ( passthrough ) { base.truncate(id) ; return ; } - if ( id > otherAllocOffset ) - { - transObjects.truncate(mapToOther(id)) ; - return ; - } - base.truncate(id) ; - transObjects.truncate(0) ; - otherAllocOffset = base.length() ; - } - - @Override - public Block allocWrite(int maxBytes) - { - if ( passthrough ) return base.allocWrite(maxBytes) ; - Block block = transObjects.allocWrite(maxBytes) ; - block = new Block(block.getId()+otherAllocOffset, block.getByteBuffer()) ; - return block ; - } - - @Override - public void completeWrite(Block block) - { - if ( passthrough ) { base.completeWrite(block) ; return ; } - block = new Block(block.getId()-otherAllocOffset, block.getByteBuffer()) ; - transObjects.completeWrite(block) ; - } - - @Override - public void abortWrite(Block block) - { - if ( passthrough ) { base.abortWrite(block) ; return ; } - block = new Block(block.getId()-otherAllocOffset, block.getByteBuffer()) ; - transObjects.abortWrite(block) ; - } - - /** Convert from a id to the id in the "other" file */ - private long mapToOther(long x) { return x-otherAllocOffset ; } - /** Convert from a id in other to an external id */ - private long mapFromOther(long x) { return x+otherAllocOffset ; } - - @Override - public long write(ByteBuffer buffer) - { - if ( passthrough ) { return base.write(buffer) ; } - // Write to auxillary - long x = transObjects.write(buffer) ; - return mapFromOther(x) ; - } - - @Override - public ByteBuffer read(long id) - { - if ( passthrough ) { return base.read(id) ; } - if ( id < otherAllocOffset ) - return base.read(id) ; - long x = mapToOther(id) ; - return transObjects.read(id-otherAllocOffset) ; +import org.apache.jena.tdb.base.objectfile.ObjectFileWrapper ; + +/** Add transactionality control to an ObjectFile. + * ObjectFiles are "append only" so with a single rwiter environment, + * we just need to manage a reset on abort. + * A crash in a transaction will accumulate some junk in the file. + * This is now a trandeoff of speed and space. + * + * Speed : append to the original file directly and tolerate junk. + * + * Space : use a journal file and write to main file on commit. + * + * @see ObjectFileTransComplex + */ +class ObjectFileTrans extends ObjectFileWrapper implements TransactionLifecycle { + ObjectFileTrans(Transaction txn /*unused*/, ObjectFile other) { + super(other); } - @Override - public long length() - { - if ( passthrough ) { return base.length() ; } - return otherAllocOffset+transObjects.length() ; - } + private long start = 0; @Override - public boolean isEmpty() - { - if ( passthrough ) { return base.isEmpty() ; } - return transObjects.isEmpty() && base.isEmpty() ; + public void begin(Transaction txn) { + start = other.length(); } @Override - public Iterator> all() - { - if ( passthrough ) { return base.all() ; } - return Iter.concat(base.all(), transObjects.all()) ; + public void abort(Transaction txn) { + other.truncate(start); } @Override - public void sync() - { - if ( passthrough ) { base.sync() ; return ; } + public void commitPrepare(Transaction txn) { + // Sync early - before the journal, with its index blocks, is committed. + other.sync(); } @Override - public void close() - { - if ( passthrough ) { base.close() ; return ; } - } + public void commitEnact(Transaction txn) { } @Override - public String getLabel() - { - return "("+base.getLabel()+":"+transObjects.getLabel()+")" ; - } -} + public void commitClearup(Transaction txn) {} +} \ No newline at end of file diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTransComplex.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTransComplex.java new file mode 100644 index 00000000000..59b7d83fab6 --- /dev/null +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/ObjectFileTransComplex.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.tdb.transaction; + +import java.nio.ByteBuffer ; +import java.util.Iterator ; + +import org.apache.jena.atlas.iterator.Iter ; +import org.apache.jena.atlas.lib.Pair ; +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.atlas.logging.Log ; +import org.apache.jena.tdb.base.block.Block ; +import org.apache.jena.tdb.base.file.FileException ; +import org.apache.jena.tdb.base.objectfile.ObjectFile ; + +/** + * This version works with two files - the main, base file and a per-transction journal + * file. It writes to the journal side. + * + * To flush, it writes the journal to the base file. + * + * It is relatively untested. + * + * Compare to {@link ObjectFileTrans} uses the fact that object files are append only so + * writes directly and resets on abort. + * + * @see ObjectFileTrans + */ +public class ObjectFileTransComplex implements ObjectFile, TransactionLifecycle +{ + private final ObjectFile transObjects ; + private long otherAllocOffset ; // record where we start allocating + private boolean passthrough = false ; + private boolean inTransaction = false ; + private final ObjectFile base ; + + // For recovery replay, we need to truncate "base" first. + + public ObjectFileTransComplex(Transaction txn, ObjectFile base, ObjectFile other) + { + // The "other" object file must use the same allocation policy. + this.base = base ; + this.transObjects = other ; + inTransaction = false ; + + // [TxTDB:PATCH-UP] Begin is not being called. + this.otherAllocOffset = base.length() ; + //Log.info(this, getLabel()+": otherAllocOffset = "+otherAllocOffset) ; + } + + // Begin read ==> passthrough. + + @Override + public void begin(Transaction txn) + { + passthrough = false ; + inTransaction = true ; + transObjects.reposition(0) ; + this.otherAllocOffset = base.length() ; + } + + @Override + public void commitPrepare(Transaction txn) + { + if ( ! inTransaction ) + throw new TDBTransactionException("Not in a transaction for a commit to happen") ; + transObjects.sync() ; + } + + @Override + public void commitEnact(Transaction txn) + { + if ( ! inTransaction ) + throw new TDBTransactionException("Not in a transaction for a commit to happen") ; + append() ; + base.sync() ; + transObjects.reposition(0) ; + } + + @Override + public void abort(Transaction txn) + { + transObjects.reposition(0) ; + } + + @Override + public void commitClearup(Transaction txn) + { + transObjects.truncate(0) ; + passthrough = true ; + } + + /** Copy from the temporary file to the real file */ + private void append() + { + // We could write directly to the real file if: + // we record the truncate point needed for an abort + // manage partial final writes + // deny the existence of nodes after the transaction mark. + // Later - stay simple for now. + + // Truncate/position the ObjectFile. + base.reposition(otherAllocOffset) ; + + Iterator> iter = transObjects.all() ; + for ( ; iter.hasNext() ; ) + { + Pair p = iter.next() ; + String s = StrUtils.fromUTF8bytes(p.getRight().array()) ; + + long x = base.write(p.getRight()) ; + + if ( p.getLeft()+otherAllocOffset != x ) + throw new FileException("Expected id of "+(p.getLeft()+otherAllocOffset)+", got an id of "+x) ; + } + } + + //public void setPassthrough(boolean v) { passthrough = v ; } + + @Override + public void reposition(long id) + { + if ( passthrough ) { base.reposition(id) ; return ; } + if ( id > otherAllocOffset ) + { + transObjects.reposition(mapToOther(id)) ; + return ; + } + + Log.warn(this, "Unexpected: Attempt to reposition over base file") ; + base.reposition(id) ; + transObjects.reposition(0) ; + otherAllocOffset = base.length() ; + } + + @Override + public void truncate(long id) + { + if ( passthrough ) { base.truncate(id) ; return ; } + if ( id > otherAllocOffset ) + { + transObjects.truncate(mapToOther(id)) ; + return ; + } + base.truncate(id) ; + transObjects.truncate(0) ; + otherAllocOffset = base.length() ; + } + + @Override + public Block allocWrite(int maxBytes) + { + if ( passthrough ) return base.allocWrite(maxBytes) ; + Block block = transObjects.allocWrite(maxBytes) ; + block = new Block(block.getId()+otherAllocOffset, block.getByteBuffer()) ; + return block ; + } + + @Override + public void completeWrite(Block block) + { + if ( passthrough ) { base.completeWrite(block) ; return ; } + block = new Block(block.getId()-otherAllocOffset, block.getByteBuffer()) ; + transObjects.completeWrite(block) ; + } + + @Override + public void abortWrite(Block block) + { + if ( passthrough ) { base.abortWrite(block) ; return ; } + block = new Block(block.getId()-otherAllocOffset, block.getByteBuffer()) ; + transObjects.abortWrite(block) ; + } + + /** Convert from a id to the id in the "other" file */ + private long mapToOther(long x) { return x-otherAllocOffset ; } + /** Convert from a id in other to an external id */ + private long mapFromOther(long x) { return x+otherAllocOffset ; } + + @Override + public long write(ByteBuffer buffer) + { + if ( passthrough ) { return base.write(buffer) ; } + // Write to auxillary + long x = transObjects.write(buffer) ; + return mapFromOther(x) ; + } + + @Override + public ByteBuffer read(long id) + { + if ( passthrough ) { return base.read(id) ; } + if ( id < otherAllocOffset ) + return base.read(id) ; + long x = mapToOther(id) ; + return transObjects.read(id-otherAllocOffset) ; + } + + @Override + public long length() + { + if ( passthrough ) { return base.length() ; } + return otherAllocOffset+transObjects.length() ; + } + + @Override + public boolean isEmpty() + { + if ( passthrough ) { return base.isEmpty() ; } + return transObjects.isEmpty() && base.isEmpty() ; + } + + @Override + public Iterator> all() + { + if ( passthrough ) { return base.all() ; } + return Iter.concat(base.all(), transObjects.all()) ; + } + + @Override + public void sync() + { + if ( passthrough ) { base.sync() ; return ; } + } + + @Override + public void close() + { + if ( passthrough ) { base.close() ; return ; } + } + + @Override + public String getLabel() + { + return "("+base.getLabel()+":"+transObjects.getLabel()+")" ; + } +} diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/Transaction.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/Transaction.java index 5bfd6f07cc1..61859f8dead 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/Transaction.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/Transaction.java @@ -38,7 +38,7 @@ public class Transaction private final Journal journal ; private final ReadWrite mode ; - private final List nodeTableTrans = new ArrayList<>() ; + private final List objectFileTrans = new ArrayList<>() ; private final List blkMgrs = new ArrayList<>() ; // The dataset this is a transaction over - may be a commited, pending dataset. private final DatasetGraphTDB basedsg ; @@ -171,9 +171,9 @@ private boolean isIOException(Throwable ex) { private void prepare() { state = TxnState.PREPARING ; - for ( BlockMgrJournal x : blkMgrs ) + for ( TransactionLifecycle x : objectFileTrans ) x.commitPrepare(this) ; - for ( NodeTableTrans x : nodeTableTrans ) + for ( TransactionLifecycle x : blkMgrs ) x.commitPrepare(this) ; } @@ -189,10 +189,9 @@ public void abort() { throw new TDBTransactionException("Transaction has already committed or aborted") ; try { // Clearup. - for ( BlockMgrJournal x : blkMgrs ) + for ( TransactionLifecycle x : objectFileTrans ) x.abort(this) ; - - for ( NodeTableTrans x : nodeTableTrans ) + for ( TransactionLifecycle x : blkMgrs ) x.abort(this) ; } catch (RuntimeException ex) { @@ -314,13 +313,13 @@ public void removeIterator(Iterator iter) { /** Return the list of items registered for the transaction lifecycle */ public List lifecycleComponents() { List x = new ArrayList<>() ; - x.addAll(nodeTableTrans) ; + x.addAll(objectFileTrans) ; x.addAll(blkMgrs) ; return x ; } - /*package*/ void addComponent(NodeTableTrans ntt) { - nodeTableTrans.add(ntt) ; + /*package*/ void addComponent(ObjectFileTrans oft) { + objectFileTrans.add(oft); } /*package*/ void addComponent(BlockMgrJournal blkMgr) { diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/extra/T_QuadsObjectIsNull.java b/jena-tdb/src/test/java/org/apache/jena/tdb/extra/T_QuadsObjectIsNull.java deleted file mode 100644 index ca701dfc988..00000000000 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/extra/T_QuadsObjectIsNull.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.tdb.extra ; - -import java.util.Iterator ; - -import org.apache.jena.atlas.lib.FileOps ; -import org.apache.jena.atlas.logging.LogCtl ; -import org.apache.jena.query.ReadWrite ; -import org.apache.jena.sparql.core.Quad ; -import org.apache.jena.sparql.sse.SSE ; -import org.apache.jena.tdb.TDBFactory ; -import org.apache.jena.tdb.base.file.Location ; -import org.apache.jena.tdb.sys.SystemTDB ; -import org.apache.jena.tdb.transaction.DatasetGraphTransaction ; -import org.apache.jena.tdb.transaction.Journal ; -import org.apache.jena.tdb.transaction.JournalControl ; -import org.apache.jena.tdb.transaction.NodeTableTrans ; - -public class T_QuadsObjectIsNull { -// static { -// ARQ.getContext().set(SystemTDB.symFileMode, "direct"); -// TDB.getContext().set(TDB.symUnionDefaultGraph, true); -// } - - static String DIR = "DBX" ; - static Location location = Location.create(DIR) ; - - public static void main(String[] args) { - - if ( false ) - { - LogCtl.enable(SystemTDB.syslog.getName()) ; - LogCtl.enable(Journal.class) ; - LogCtl.enable(JournalControl.class) ; - LogCtl.enable(NodeTableTrans.class) ; - } - if ( false ) - { - String journal = "DBX/journal.jrnl" ; - if ( FileOps.exists(journal)) - JournalControl.print(journal) ; - } - - if ( false ) { - FileOps.ensureDir(DIR) ; - FileOps.clearDirectory(DIR) ; - } - one() ; - } - - public static void write(DatasetGraphTransaction dsg, Quad quad) - { - dsg.begin(ReadWrite.WRITE) ; - dsg.add(quad) ; - if ( ! dsg.contains(quad) ) - throw new RuntimeException("No quad: "+quad) ; - dsg.commit() ; - dsg.end() ; - } - - private static void dump(DatasetGraphTransaction dsg) - { - dsg.begin(ReadWrite.READ); - Iterator iter = dsg.find() ; - for ( ; iter.hasNext() ; ) - { - Quad q = iter.next() ; - System.out.println(q) ; - } - //RiotWriter.writeNQuads(System.out, dsg) ; - dsg.commit(); - dsg.end(); - } - - public static void one() - { - Quad q1 = SSE.parseQuad("( '1')") ; - Quad q2 = SSE.parseQuad("( '2')") ; - Quad q3 = SSE.parseQuad("( '3')") ; - - DatasetGraphTransaction dsg = (DatasetGraphTransaction)TDBFactory.createDatasetGraph(location); - System.out.println("Start") ; - dump(dsg) ; - - write(dsg, q1) ; - write(dsg, q2) ; - //write(dsg, q3) ; - System.out.println("Finish") ; - dump(dsg) ; - } - - -} diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestNodeTableTrans.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestNodeTableTrans.java deleted file mode 100644 index c676c39e239..00000000000 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestNodeTableTrans.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.tdb.transaction; - -import org.apache.jena.atlas.junit.BaseTest ; -import org.apache.jena.graph.Node ; -import org.apache.jena.query.ReadWrite ; -import org.apache.jena.sparql.util.NodeFactoryExtra ; -import org.apache.jena.tdb.base.file.Location ; -import org.apache.jena.tdb.base.objectfile.ObjectFile ; -import org.apache.jena.tdb.base.record.RecordFactory ; -import org.apache.jena.tdb.index.Index ; -import org.apache.jena.tdb.index.IndexMap ; -import org.apache.jena.tdb.setup.Build ; -import org.apache.jena.tdb.store.NodeId ; -import org.apache.jena.tdb.store.nodetable.NodeTable ; -import org.apache.jena.tdb.sys.SystemTDB ; -import org.apache.jena.tdb.transaction.NodeTableTrans ; -import org.apache.jena.tdb.transaction.Transaction ; -import org.junit.Test ; - -public abstract class AbstractTestNodeTableTrans extends BaseTest -{ - abstract protected ObjectFile createObjectFile() ; - abstract protected Location getLocation() ; - - private NodeTableTrans create(Transaction txn, Node...nodes) - { - NodeTable base = Build.makeNodeTable(getLocation()) ; - for ( Node n : nodes ) - base.getAllocateNodeId(n) ; - return create(txn, base) ; - } - - private NodeTableTrans create(Transaction txn, NodeTable base) - { - RecordFactory recordFactory = new RecordFactory(SystemTDB.LenNodeHash, SystemTDB.SizeOfNodeId) ; - Index idx = new IndexMap(recordFactory) ; - ObjectFile objectFile = createObjectFile() ; - NodeTableTrans ntt = new NodeTableTrans(txn, "test", base, idx, objectFile) ; - return ntt ; - } - - protected static Node node1 = NodeFactoryExtra.parseNode("") ; - protected static Node node2 = NodeFactoryExtra.parseNode("") ; - protected static Node node3 = NodeFactoryExtra.parseNode("") ; - - static void contains(NodeTable nt, Node...nodes) - { - for ( Node n : nodes) - { - NodeId nodeId = nt.getNodeIdForNode(n) ; - assertFalse(NodeId.isDoesNotExist(nodeId)) ; - } - } - - Transaction createTxn(long id) - { - return new Transaction(null, 99, ReadWrite.WRITE, id, null, null) ; - } - - @Test public void nodetrans_01() - { - Transaction txn = createTxn(11) ; - NodeTableTrans ntt = create(txn) ; - ntt.begin(txn) ; - ntt.abort(txn) ; - } - - - @Test public void nodetrans_02() - { - Transaction txn = createTxn(11) ; - NodeTableTrans ntt = create(txn) ; - NodeTable nt0 = ntt.getBaseNodeTable() ; - - ntt.begin(txn) ; - // Add a node - NodeId nodeId = ntt.getAllocateNodeId(node1) ; - // Check not in the base. - assertNull(nt0.getNodeForNodeId(nodeId)) ; - // Check is in the transaction node table. - assertEquals(NodeId.NodeDoesNotExist, nt0.getNodeIdForNode(node1)) ; - assertEquals(node1, ntt.getNodeForNodeId(nodeId)) ; - - ntt.commitPrepare(txn) ; - ntt.commitEnact(txn) ; - // Check it is now in the base. - assertEquals(node1, nt0.getNodeForNodeId(nodeId)) ; - assertEquals(nodeId, nt0.getNodeIdForNode(node1)) ; - ntt.commitClearup(txn) ; - } - - @Test public void nodetrans_03() - { - Transaction txn = createTxn(11) ; - NodeTableTrans ntt = create(txn) ; - NodeTable nt0 = ntt.getBaseNodeTable() ; - - ntt.begin(txn) ; - // Add a node - NodeId nodeId = ntt.getAllocateNodeId(node1) ; - // Check not in the base. - assertEquals(NodeId.NodeDoesNotExist, nt0.getNodeIdForNode(node1)) ; - assertNull(nt0.getNodeForNodeId(nodeId)) ; - // Check is in the transaction node table. - assertEquals(node1, ntt.getNodeForNodeId(nodeId)) ; - - ntt.abort(txn) ; - // Check it is not in the base. - assertEquals(NodeId.NodeDoesNotExist, nt0.getNodeIdForNode(node1)) ; - assertNull(nt0.getNodeForNodeId(nodeId)) ; - ntt.commitClearup(txn) ; - } - - @Test public void nodetrans_04() - { - Transaction txn = createTxn(11) ; - NodeTableTrans ntt = create(txn, node1) ; - NodeTable nt0 = ntt.getBaseNodeTable() ; - ntt.begin(txn) ; - // Add a node - NodeId nodeId = ntt.getAllocateNodeId(node2) ; - // Not here - assertEquals(NodeId.NodeDoesNotExist, nt0.getNodeIdForNode(node2)) ; - // Is here - assertEquals(nodeId, ntt.getNodeIdForNode(node2)) ; - ntt.commitPrepare(txn) ; - ntt.commitEnact(txn) ; - assertEquals(nodeId, nt0.getNodeIdForNode(node2)) ; - ntt.commitClearup(txn) ; - } - - - @Test - public void nodetrans_05() - { - // 2 transactions - no blocking reader - create a second NodeTableTrans - Transaction txn1 = createTxn(11) ; - NodeTableTrans ntt1 = create(txn1, node1) ; - NodeId nodeId1 = ntt1.getBaseNodeTable().getNodeIdForNode(node1) ; - - ntt1.begin(txn1) ; - NodeId nodeId2 = ntt1.getAllocateNodeId(node2) ; - ntt1.commitPrepare(txn1) ; - ntt1.commitEnact(txn1) ; - ntt1.commitClearup(txn1) ; - - Transaction txn2 = createTxn(12) ; - NodeTableTrans ntt2 = create(txn2, ntt1.getBaseNodeTable()) ; - ntt2.begin(txn2) ; - assertEquals(nodeId1, ntt2.getNodeIdForNode(node1)) ; - assertEquals(nodeId2, ntt2.getNodeIdForNode(node2)) ; - NodeId nodeId3 = ntt2.getAllocateNodeId(node3) ; - assertEquals(nodeId3, ntt2.getNodeIdForNode(node3)) ; - ntt2.commitPrepare(txn2) ; - ntt2.commitEnact(txn2) ; - ntt2.commitClearup(txn2) ; - - assertEquals(nodeId1, ntt1.getBaseNodeTable().getNodeIdForNode(node1)) ; - assertEquals(nodeId2, ntt1.getBaseNodeTable().getNodeIdForNode(node2)) ; - assertEquals(nodeId3, ntt1.getBaseNodeTable().getNodeIdForNode(node3)) ; - } - - @Test - public void nodetrans_06() - { - // 2 transactions - blocking reader - create a second NodeTableTrans - Transaction txn1 = createTxn(11) ; - NodeTableTrans ntt1 = create(txn1, node1) ; - NodeId nodeId1 = ntt1.getBaseNodeTable().getNodeIdForNode(node1) ; - - ntt1.begin(txn1) ; - NodeId nodeId2 = ntt1.getAllocateNodeId(node2) ; - ntt1.commitPrepare(txn1) ; - - // READ - don't enact - Transaction txn2 = createTxn(12) ; - NodeTableTrans ntt2 = create(txn2, ntt1.getBaseNodeTable()) ; - ntt2.begin(txn2) ; - assertEquals(nodeId1, ntt2.getNodeIdForNode(node1)) ; - assertEquals(nodeId2, ntt2.getNodeIdForNode(node2)) ; - - NodeId nodeId3 = ntt2.getAllocateNodeId(node3) ; - assertEquals(nodeId3, ntt2.getNodeIdForNode(node3)) ; - ntt2.commitPrepare(txn2) ; - - - // READ ends. - - ntt1.commitEnact(txn1) ; - ntt1.commitClearup(txn1) ; - - ntt2.commitEnact(txn2) ; - ntt2.commitClearup(txn2) ; - - assertEquals(nodeId1, ntt1.getBaseNodeTable().getNodeIdForNode(node1)) ; - assertEquals(nodeId2, ntt1.getBaseNodeTable().getNodeIdForNode(node2)) ; - assertEquals(nodeId3, ntt1.getBaseNodeTable().getNodeIdForNode(node3)) ; - } - - @Test - public void nodetrans_07() - { - Transaction txn = createTxn(13) ; - NodeTableTrans ntt = create(txn) ; - - ntt.begin(txn) ; - assertEquals(NodeId.NodeIdAny, ntt.getNodeIdForNode(Node.ANY)) ; - assertEquals(Node.ANY, ntt.getNodeForNodeId(NodeId.NodeIdAny)) ; - - assertTrue(ntt.isEmpty()) ; - - ntt.commitPrepare(txn) ; - ntt.commitEnact(txn) ; - ntt.commitClearup(txn) ; - } - -} diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTrans.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTrans.java index 49dc7ee93bf..dbbcf7d246a 100644 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTrans.java +++ b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTrans.java @@ -37,7 +37,6 @@ public abstract class AbstractTestObjectFileTrans extends BaseTest { static long count = 0 ; ObjectFile file1 ; - ObjectFile file2 ; ObjectFileTrans file ; Transaction txn ; @@ -51,7 +50,7 @@ public void setup() { txn = new Transaction(null, 5, ReadWrite.WRITE, ++count, null, tm) ; file1 = createFile("base") ; - file2 = createFile("log") ; + //file2 = createFile("log") ; } @After @@ -83,7 +82,10 @@ private static void contains(ObjectFile f, String... contents) } - private void init() { file = new ObjectFileTrans(null, file1, file2) ; } + private void init() { + //file = new ObjectFileTransComplex(null, file1, file2) ; + file = new ObjectFileTrans(null, file1) ; + } static void fill(ObjectFile file, String... contents) { @@ -103,7 +105,7 @@ static void fill(ObjectFile file, String... contents) init() ; file.begin(txn) ; - contains(file2) ; + //contains(file2) ; file.commitPrepare(txn) ; file.commitEnact(txn) ; contains(file1, "ABC") ; diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTransComplex.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTransComplex.java new file mode 100644 index 00000000000..0e73ca2e0de --- /dev/null +++ b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/AbstractTestObjectFileTransComplex.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.tdb.transaction; + +import java.nio.ByteBuffer ; +import java.util.Iterator ; + +import org.apache.jena.atlas.junit.BaseTest ; +import org.apache.jena.atlas.lib.Pair ; +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.query.ReadWrite ; +import org.apache.jena.tdb.base.objectfile.ObjectFile ; +import org.junit.After ; +import org.junit.Before ; +import org.junit.Test ; + +/** Clone of AbstractTestObjectFileTrans that understands 2-file ObjectFileTransComplex */ +public abstract class AbstractTestObjectFileTransComplex extends BaseTest +{ + static long count = 0 ; + ObjectFile file1 ; + ObjectFile file2 ; + ObjectFileTransComplex file ; + Transaction txn ; + + abstract ObjectFile createFile(String basename) ; + abstract void deleteFile(String basename) ; + + TransactionManager tm = null ; + + @Before + public void setup() + { + txn = new Transaction(null, 5, ReadWrite.WRITE, ++count, null, tm) ; + file1 = createFile("base") ; + file2 = createFile("log") ; + } + + @After + public void teardown() + { + deleteFile("base") ; + deleteFile("log") ; + } + + static void write(ObjectFile file, String str) + { + byte b[] = StrUtils.asUTF8bytes(str) ; + ByteBuffer bb = ByteBuffer.wrap(b) ; + file.write(bb) ; + } + + private static void contains(ObjectFile f, String... contents) + { + Iterator> iter = f.all() ; + for ( String s : contents ) + { + assertTrue(iter.hasNext()) ; + Pair p = iter.next() ; + String s2 = StrUtils.fromUTF8bytes(p.cdr().array()) ; + assertEquals(s, s2) ; + } + + assertFalse(iter.hasNext()) ; + } + + + private void init() { + file = new ObjectFileTransComplex(null, file1, file2) ; + } + + static void fill(ObjectFile file, String... contents) + { + for ( String s : contents ) + write(file, s) ; + } + + @Test public void objFileTrans_01() + { + init() ; + contains(file) ; + } + + @Test public void objFileTrans_02() + { + fill(file1, "ABC") ; + init() ; + + file.begin(txn) ; + // Test empty. + contains(file2) ; + file.commitPrepare(txn) ; + file.commitEnact(txn) ; + contains(file1, "ABC") ; + file.commitClearup(txn) ; + } + + @Test public void objFileTrans_03() + { + fill(file1, "ABC") ; + init() ; + file.begin(txn) ; + write(file, "X") ; + file.commitPrepare(txn) ; + file.commitEnact(txn) ; + contains(file1, "ABC", "X") ; + file.commitClearup(txn) ; + } + + @Test public void objFileTrans_04() + { + fill(file1, "ABC", "ABC") ; + init() ; + file.begin(txn) ; + write(file, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") ; + file.commitPrepare(txn) ; + file.commitEnact(txn) ; + contains(file1, "ABC", "ABC", "ABCDEFGHIJKLMNOPQRSTUVWXYZ") ; + file.commitClearup(txn) ; + } + + @Test public void objFileTrans_05() + { + fill(file1, "ABC") ; + init() ; + file.begin(txn) ; + write(file, "ABCDEF") ; + file.abort(txn) ; + contains(file1, "ABC") ; + file.commitClearup(txn) ; + } + + @Test public void objFileTrans_06() + { + fill(file1, "ABC", "123") ; + init() ; + file.begin(txn) ; + write(file, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") ; + file.abort(txn) ; + contains(file1, "ABC", "123") ; + } +} diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TS_TransactionTDB.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TS_TransactionTDB.java index 87bbf1e99c0..08abf991f4f 100644 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TS_TransactionTDB.java +++ b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TS_TransactionTDB.java @@ -31,8 +31,11 @@ , TestTransIterator.class , TestObjectFileTransMem.class , TestObjectFileTransStorage.class - , TestNodeTableTransMem.class - , TestNodeTableTransDisk.class + + // ObjectFileTransComplex is not used by TDB currently. + , TestObjectFileTransComplexMem.class + , TestObjectFileTransComplexStorage.class + , TestTransMem.class , TestTransDiskDirect.class , TestTransDiskMapped.class diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransMem.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexMem.java similarity index 76% rename from jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransMem.java rename to jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexMem.java index 91b5028f9df..3ea9d5a5bc1 100644 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransMem.java +++ b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexMem.java @@ -19,20 +19,18 @@ package org.apache.jena.tdb.transaction; import org.apache.jena.tdb.base.file.FileFactory ; -import org.apache.jena.tdb.base.file.Location ; import org.apache.jena.tdb.base.objectfile.ObjectFile ; -public class TestNodeTableTransMem extends AbstractTestNodeTableTrans +public class TestObjectFileTransComplexMem extends AbstractTestObjectFileTransComplex { @Override - protected ObjectFile createObjectFile() + ObjectFile createFile(String basename) { - return FileFactory.createObjectFileMem("mem-trans") ; + return FileFactory.createObjectFileMem("test") ; } @Override - protected Location getLocation() - { - return Location.mem() ; - } + void deleteFile(String basename) + {} + } diff --git a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransDisk.java b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexStorage.java similarity index 50% rename from jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransDisk.java rename to jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexStorage.java index 48fdd15d280..809cbe1846c 100644 --- a/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestNodeTableTransDisk.java +++ b/jena-tdb/src/test/java/org/apache/jena/tdb/transaction/TestObjectFileTransComplexStorage.java @@ -18,34 +18,36 @@ package org.apache.jena.tdb.transaction; -import org.apache.jena.tdb.ConfigTest ; -import org.apache.jena.tdb.base.file.FileSet ; -import org.apache.jena.tdb.base.file.Location ; +import org.apache.jena.tdb.base.file.BufferChannel ; +import org.apache.jena.tdb.base.file.BufferChannelMem ; import org.apache.jena.tdb.base.objectfile.ObjectFile ; -import org.apache.jena.tdb.sys.SetupTDB ; -import org.junit.Before ; +import org.apache.jena.tdb.base.objectfile.ObjectFileStorage ; -public class TestNodeTableTransDisk extends AbstractTestNodeTableTrans +public class TestObjectFileTransComplexStorage extends AbstractTestObjectFileTransComplex { - Location loc = null ; - static int count = 0 ; - - @Before public void before() - { - String dir = ConfigTest.getCleanDir() ; - loc = Location.create(dir) ; - } - @Override - protected ObjectFile createObjectFile() + ObjectFile createFile(String basename) { - FileSet fs = new FileSet(loc, "data") ; - return SetupTDB.makeObjectFile(fs) ; +// String dir = ConfigTest.getTestingDir() ; +// Location loc = Location.create(dir) ; +// String fn = loc.getPath(basename) ; +// FileOps.deleteSilent(fn) ; +// BufferChannel chan = new BufferChannelFile(fn) ; +// return new ObjectFileStorage(chan) ; + + BufferChannel chan = BufferChannelMem.create() ; + // Small buffer + return new ObjectFileStorage(chan,10) ; + } + @Override - protected Location getLocation() + void deleteFile(String basename) { - return loc ; +// String dir = ConfigTest.getTestingDir() ; +// Location loc = Location.create(dir) ; +// String fn = loc.getPath(basename) ; +// FileOps.delete(fn) ; } } From afc9b8c038eff18c3654d47e112a33b3a653f66f Mon Sep 17 00:00:00 2001 From: Andy Seaborne Date: Sat, 5 Aug 2017 21:48:22 +0100 Subject: [PATCH 2/2] JENA-1379: Check for old-style dat-jrnl files --- .../jena/tdb/transaction/JournalControl.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java index 03a5d32ef52..748149efb14 100644 --- a/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java +++ b/jena-tdb/src/main/java/org/apache/jena/tdb/transaction/JournalControl.java @@ -85,11 +85,9 @@ public static void recovery(DatasetGraphTDB dsg) if ( journal == null || journal.isEmpty() ) return ; - for ( FileRef fileRef : dsg.getConfig().objectFiles.keySet() ) - recoverNodeDat(dsg, fileRef) ; recoverFromJournal(dsg.getConfig(), journal) ; - journal.close() ; + journal.close(); // Recovery complete. Tidy up. Node journal files have already been handled. if ( journal.getFilename() != null ) { @@ -120,6 +118,9 @@ public static boolean recoverFromJournal(StorageConfig sConf, Journal jrnl) { if ( jrnl.isEmpty() ) return false ; + + for ( FileRef fileRef : sConf.objectFiles.keySet() ) + recoverNodeDat(sConf.location, fileRef) ; long posn = 0 ; for ( ;; ) @@ -194,20 +195,23 @@ private static void recoverSegment(Journal jrnl, long startPosn, long endPosn, S * Node data files are append-only so recovering. * This code is only for ObjectFileTransComplex. */ - private static void recoverNodeDat(DatasetGraphTDB dsg, FileRef fileRef) + private static void recoverNodeDat(Location loc, FileRef fileRef) { -// // See DatasetBuilderTxn - same name generation code. -// String objFilename = fileRef.getFilename()+"-"+Names.extJournal ; -// objFilename = dsg.getLocation().absolute(objFilename) ; -// File jrnlFile = new File(objFilename) ; -// if ( jrnlFile.exists() && jrnlFile.length() > 0 ) -// { -// syslog.info("Clearing node data: "+fileRef.getFilename()) ; -// ObjectFile dataJrnl = FileFactory.createObjectFileDisk(objFilename) ; -// dataJrnl.truncate(0); -// } -// if ( jrnlFile.exists() ) -// FileOps.delete(objFilename) ; + // See DatasetBuilderTxn (Jena 3.4.0 or earlier) - same name generation code. + String objFilename = fileRef.getFilename()+"-"+Names.extJournal ; + objFilename = loc.absolute(objFilename) ; + File jrnlFile = new File(objFilename) ; + if ( jrnlFile.exists() ) { + if ( jrnlFile.length() > 0 ) { + syslog.info("Found dat-jrnl file : earlier version of Jena"+fileRef.getFilename()) ; + syslog.info(" To clearup: run TDB from a version of Jena 3.0.0-3.4.0"); + syslog.info(" dat-jrnl should then go away"); + syslog.info(" See https://issues.apache.org/jira/browse/JENA-1379"); + throw new TDBException("Manual recovery required - see log - see JENA-1379 "); + } + //Empty - nothing to do anyway - clearup. + FileOps.delete(objFilename) ; + } } public static void replay(Transaction transaction)