Permalink
Browse files

Changes to ignore the secondary Indexes, because cassandra will rebuild

once the Stream is complete.
  • Loading branch information...
Vijay2win committed Jun 6, 2012
1 parent b759ee7 commit f94e4abfaaaba3a7d926f4074e89e17d60e1ed20
@@ -20,6 +20,7 @@
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
+import java.util.regex.Pattern;
/*
* Incremental SSTable Restore using SSTable Loader
@@ -29,6 +30,10 @@
{
private static final Logger logger = LoggerFactory.getLogger(IncrementalRestore.class);
public static final String JOBNAME = "INCR_RESTORE_THREAD";
+
+ /* Marked public for testing */
+ public static final Pattern SECONDRY_INDEX_PATTERN = Pattern.compile("^[a-zA-Z_0-9-]+\\.[a-zA-Z_0-9-]+\\.[a-z1-9]{2,4}$");
+
private final File restoreDir;
@Inject
@@ -66,12 +71,15 @@ public void execute() throws Exception
{
AbstractBackupPath temp = incrementals.next();
if (tracker.contains(temp) || start.compareTo(temp.time) >= 0)
- continue;
+ continue; // ignore the ones which where already downloaded.
if (temp.getType() != BackupFileType.SST)
- continue;
- // skip System informations.
+ continue; // download SST's only.
+ // skip System Keyspace, else you will run into concurrent schema issues.
if (temp.getKeyspace().equalsIgnoreCase("System"))
continue;
+ /* Cassandra will rebuild Secondary index's after streaming is complete so we can ignore those */
+ if (SECONDRY_INDEX_PATTERN.matcher(temp.fileName).matches()) // Make this use the constant from 1.1
+ continue;
File keyspaceDir = new File(restoreDir, temp.keyspace);
FileUtils.createDirectory(keyspaceDir);
download(temp, new File(keyspaceDir, temp.fileName));
@@ -88,7 +96,7 @@ public void execute() throws Exception
}
/**
- * Run every 10 Sec
+ * Run every 20 Sec
*/
public static TaskTimer getTimer()
{
@@ -17,6 +17,7 @@
import com.netflix.priam.aws.S3BackupPath;
import com.netflix.priam.backup.AbstractBackupPath;
import com.netflix.priam.backup.BRTestModule;
+import com.netflix.priam.backup.IncrementalRestore;
import com.netflix.priam.identity.InstanceIdentity;
import com.netflix.priam.utils.FifoQueue;
@@ -78,7 +79,18 @@ public void testAbstractPath()
path.parseRemote("test_backup/fake-region/fakecluster/123456/201108260000/SNAP/ks1/f326.db To: cass/data/ks1/f326.db");
Assert.assertEquals(path, queue.first());
- // for (AbstractBackupPath p : queue)
- // System.out.println(p);
}
+
+ @Test
+ public void testIgnoreIndexFiles()
+ {
+ String[] testInputs = new String[] { "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Digest.sha1",
+ "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Filter.db", "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Data.db",
+ "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Statistics.db", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Filter.db",
+ "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Digest.sha1", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Statistics.db", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Data.db" };
+ Assert.assertFalse(IncrementalRestore.SECONDRY_INDEX_PATTERN.matcher("cfname_test_name_idx-hc.db").matches());
+ for (String input : testInputs)
+ Assert.assertTrue(IncrementalRestore.SECONDRY_INDEX_PATTERN.matcher(input).matches());
+ }
+
}
Oops, something went wrong.

0 comments on commit f94e4ab

Please sign in to comment.