-
Notifications
You must be signed in to change notification settings - Fork 2.3k
/
IncrementalBackupIT.java
248 lines (222 loc) · 9.49 KB
/
IncrementalBackupIT.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
/*
* Copyright (c) 2002-2018 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j Enterprise Edition. The included source
* code can be redistributed and/or modified under the terms of the
* GNU AFFERO GENERAL PUBLIC LICENSE Version 3
* (http://www.fsf.org/licensing/licenses/agpl-3.0.html) with the
* Commons Clause, as found in the associated LICENSE.txt file.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* Neo4j object code can be licensed independently from the source
* under separate terms from the AGPL. Inquiries can be directed to:
* licensing@neo4j.com
*
* More information is also available at:
* https://neo4j.com/licensing/
*/
package org.neo4j.backup;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestName;
import java.io.File;
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings;
import org.neo4j.ports.allocation.PortAuthority;
import org.neo4j.test.DbRepresentation;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.test.rule.SuppressOutput;
import org.neo4j.test.rule.TestDirectory;
import static org.junit.Assert.assertEquals;
public class IncrementalBackupIT
{
private final TestName testName = new TestName();
public TestDirectory testDirectory = TestDirectory.testDirectory();
public SuppressOutput suppressOutput = SuppressOutput.suppressAll();
@Rule
public RuleChain rules = RuleChain.outerRule( testName ).around( testDirectory ).around( suppressOutput );
private File serverPath;
private File backupDatabase;
private ServerInterface server;
private GraphDatabaseService db;
private File backupStore;
@Before
public void before()
{
serverPath = testDirectory.storeDir( "server" );
backupStore = testDirectory.storeDir( "backupStore" );
backupDatabase = testDirectory.databaseDir( backupStore );
}
@After
public void shutItDown() throws Exception
{
if ( server != null )
{
shutdownServer( server );
server = null;
}
if ( db != null )
{
db.shutdown();
db = null;
}
}
@Test
public void shouldDoIncrementalBackup() throws Exception
{
DbRepresentation initialDataSetRepresentation = createInitialDataSet( serverPath );
int port = PortAuthority.allocatePort();
server = startServer( serverPath, "127.0.0.1:" + port );
OnlineBackup backup = OnlineBackup.from( "127.0.0.1", port );
backup.full( backupDatabase.getPath() );
assertEquals( initialDataSetRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
DbRepresentation furtherRepresentation = addMoreData2( serverPath );
server = startServer( serverPath, "127.0.0.1:" + port );
backup.incremental( backupDatabase.getPath() );
assertEquals( furtherRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
}
@Test
public void shouldNotServeTransactionsWithInvalidHighIds() throws Exception
{
/*
* This is in effect a high level test for an edge case that happens when a relationship group is
* created and deleted in the same tx. This can end up causing an IllegalArgumentException because
* the HighIdApplier used when applying incremental updates (batch transactions in general) will postpone
* processing of added/altered record ids but deleted ids will be processed on application. This can result
* in a deleted record causing an IllegalArgumentException even though it is not the highest id in the tx.
*
* The way we try to trigger this is:
* 0. In one tx, create a node with 49 relationships, belonging to two types.
* 1. In another tx, create another relationship on that node (making it dense) and then delete all
* relationships of one type. This results in the tx state having a relationship group record that was
* created in this tx and also set to not in use.
* 2. Receipt of this tx will have the offending rel group command apply its id before the groups that are
* altered. This will try to update the high id with a value larger than what has been seen previously and
* fail the update.
* The situation is resolved by a check added in TransactionRecordState which skips the creation of such
* commands.
* Note that this problem can also happen in HA slaves.
*/
DbRepresentation initialDataSetRepresentation = createInitialDataSet( serverPath );
int port = PortAuthority.allocatePort();
server = startServer( serverPath, "127.0.0.1:" + port );
OnlineBackup backup = OnlineBackup.from( "127.0.0.1", port);
backup.full( backupDatabase.getPath() );
assertEquals( initialDataSetRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
DbRepresentation furtherRepresentation = createTransactionWithWeirdRelationshipGroupRecord( serverPath );
server = startServer( serverPath, "127.0.0.1:" + port );
backup.incremental( backupDatabase.getPath() );
assertEquals( furtherRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
}
private DbRepresentation createInitialDataSet( File path )
{
db = startGraphDatabase( path );
try ( Transaction tx = db.beginTx() )
{
db.createNode().setProperty( "name", "Goofy" );
Node donald = db.createNode();
donald.setProperty( "name", "Donald" );
Node daisy = db.createNode();
daisy.setProperty( "name", "Daisy" );
Relationship knows = donald.createRelationshipTo( daisy,
RelationshipType.withName( "LOVES" ) );
knows.setProperty( "since", 1940 );
tx.success();
}
DbRepresentation result = DbRepresentation.of( db );
db.shutdown();
return result;
}
private DbRepresentation addMoreData2( File path )
{
db = startGraphDatabase( path );
try ( Transaction tx = db.beginTx() )
{
Node donald = db.getNodeById( 2 );
Node gladstone = db.createNode();
gladstone.setProperty( "name", "Gladstone" );
Relationship hates = donald.createRelationshipTo( gladstone,
RelationshipType.withName( "HATES" ) );
hates.setProperty( "since", 1948 );
tx.success();
}
DbRepresentation result = DbRepresentation.of( db );
db.shutdown();
return result;
}
private DbRepresentation createTransactionWithWeirdRelationshipGroupRecord( File path )
{
db = startGraphDatabase( path );
int i = 0;
Node node;
RelationshipType typeToDelete = RelationshipType.withName( "A" );
RelationshipType theOtherType = RelationshipType.withName( "B" );
int defaultDenseNodeThreshold =
Integer.parseInt( GraphDatabaseSettings.dense_node_threshold.getDefaultValue() );
try ( Transaction tx = db.beginTx() )
{
node = db.createNode();
for ( ; i < defaultDenseNodeThreshold - 1; i++ )
{
node.createRelationshipTo( db.createNode(), theOtherType );
}
node.createRelationshipTo( db.createNode(), typeToDelete );
tx.success();
}
try ( Transaction tx = db.beginTx() )
{
node.createRelationshipTo( db.createNode(), theOtherType );
for ( Relationship relationship : node.getRelationships( Direction.BOTH, typeToDelete ) )
{
relationship.delete();
}
tx.success();
}
DbRepresentation result = DbRepresentation.of( db );
db.shutdown();
return result;
}
private static GraphDatabaseService startGraphDatabase( File storeDir )
{
return new TestGraphDatabaseFactory().
newEmbeddedDatabaseBuilder( storeDir ).
setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ).
setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ).
newGraphDatabase();
}
private static ServerInterface startServer( File storeDir, String serverAddress )
{
ServerInterface server = new EmbeddedServer( storeDir, serverAddress );
server.awaitStarted();
return server;
}
private static void shutdownServer( ServerInterface server ) throws Exception
{
server.shutdown();
Thread.sleep( 1000 );
}
private DbRepresentation getBackupDbRepresentation()
{
return DbRepresentation.of( backupDatabase, Config.defaults( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) );
}
}