refactor

.cproject

@@ -20,7 +20,7 @@
 							<option id="cdt.managedbuild.option.gnu.cross.prefix.639451169" name="Prefix" superClass="cdt.managedbuild.option.gnu.cross.prefix" useByScannerDiscovery="false"/>
 							<option id="cdt.managedbuild.option.gnu.cross.path.1512816234" name="Path" superClass="cdt.managedbuild.option.gnu.cross.path" useByScannerDiscovery="false"/>
 							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1350459350" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
-							<builder buildPath="${workspace_loc:/Tomahawk_rewrite}/build" id="cdt.managedbuild.builder.gnu.cross.1750567232" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/Tachyon}/build" id="cdt.managedbuild.builder.gnu.cross.1750567232" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.builder.gnu.cross"/>
 							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.773076487" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
 								<option id="gnu.c.compiler.option.optimization.level.1087624297" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.c.optimization.level.most" valueType="enumerated"/>
 								<option id="gnu.c.compiler.option.dialect.std.752679132" name="Language standard" superClass="gnu.c.compiler.option.dialect.std" useByScannerDiscovery="true" value="gnu.c.compiler.dialect.c99" valueType="enumerated"/>
@@ -51,18 +51,7 @@
 						</toolChain>
 					</folderInfo>
 					<sourceEntries>
-						<entry excluding="third_party|algorithm|algorithm/compression|algorithm/permutation|core|core/base|index|io|io/bcf|io/compression|support|io/vcf" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name="src"/>
-						<entry excluding="compression|permutation" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name="src/algorithm"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/algorithm/compression"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/algorithm/permutation"/>
-						<entry excluding="base" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/core"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/core/base"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/index"/>
-						<entry excluding="vcf|bcf|compression|support" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/io"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/io/bcf"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/io/compression"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/io/vcf"/>
-						<entry flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name="src/support"/>
+						<entry excluding="third_party" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name="src"/>
 						<entry excluding="FiniteStateEntropy/Visual|xxhash/xxhsum.c|zlib/win32|zlib/doc|zlib/msdos|zlib/as400|zlib/examples|zlib/test|zlib/contrib|FiniteStateEntropy/lib/Archives|FiniteStateEntropy/programs|zlib/qnx|third_party/zlib/old|zlib/watcom|zlib/nintendods|zlib/amiga" flags="VALUE_WORKSPACE_PATH" kind="sourcePath" name="src/third_party"/>
 					</sourceEntries>
 				</configuration>
@@ -71,15 +60,15 @@
 		</cconfiguration>
 	</storageModule>
 	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
-		<project id="Tomahawk_rewrite.null.1534595450" name="Tomahawk_rewrite"/>
+		<project id="Tachyon.null.1534595450" name="Tachyon"/>
 	</storageModule>
 	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
 	<storageModule moduleId="refreshScope" versionNumber="2">
 		<configuration configurationName="build">
-			<resource resourceType="PROJECT" workspacePath="/Tomahawk_rewrite"/>
+			<resource resourceType="PROJECT" workspacePath="/Tachyon"/>
 		</configuration>
 		<configuration configurationName="Default">
-			<resource resourceType="PROJECT" workspacePath="/Tomahawk_rewrite"/>
+			<resource resourceType="PROJECT" workspacePath="/Tachyon"/>
 		</configuration>
 	</storageModule>
 	<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets">

.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>Tomahawk_rewrite</name>
+	<name>Tachyon</name>
 	<comment></comment>
 	<projects>
 	</projects>

.settings/language.settings.xml

@@ -5,7 +5,7 @@
 			<provider copy-of="extension" id="org.eclipse.cdt.ui.UserLanguageSettingsProvider"/>
 			<provider-reference id="org.eclipse.cdt.core.ReferencedProjectsLanguageSettingsProvider" ref="shared-provider"/>
 			<provider class="org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuildCommandParser" id="org.eclipse.cdt.managedbuilder.core.GCCBuildCommandParser" keep-relative-paths="false" name="CDT GCC Build Output Parser" parameter="(g?cc)|([gc]\+\+)|(clang)" prefer-non-shared="true"/>
-			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-1884500886375924948" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${COMMAND} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
+			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-878403872703323892" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${COMMAND} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
 				<language-scope id="org.eclipse.cdt.core.gcc"/>
 				<language-scope id="org.eclipse.cdt.core.g++"/>
 			</provider>

build/src/core/subdir.mk

@@ -4,16 +4,19 @@
 
 # Add inputs and outputs from these tool invocations to the build variables 
 CPP_SRCS += \
+../src/core/BlockEntry.cpp \
 ../src/core/ImportWriter.cpp \
 ../src/core/Importer.cpp \
 ../src/core/StreamContainer.cpp 
 
 OBJS += \
+./src/core/BlockEntry.o \
 ./src/core/ImportWriter.o \
 ./src/core/Importer.o \
 ./src/core/StreamContainer.o 
 
 CPP_DEPS += \
+./src/core/BlockEntry.d \
 ./src/core/ImportWriter.d \
 ./src/core/Importer.d \
 ./src/core/StreamContainer.d 

src/algorithm/OpenHashTable.h

@@ -1,5 +1,5 @@
-#ifndef OPENHASHTABLE_H_
-#define OPENHASHTABLE_H_
+#ifndef ALGORITHM_OPENHASHTABLE_H_
+#define ALGORITHM_OPENHASHTABLE_H_
 
 #include <atomic>
 #include <cstring>
@@ -197,4 +197,4 @@ void HashTable<T, K>::clear(void){
 }
 
 
-#endif /* OPENHASHTABLE_H_ */
+#endif /* ALGORITHM_OPENHASHTABLE_H_ */

src/algorithm/compression/EncoderGenotypesRLE.h

@@ -322,8 +322,8 @@ bool EncoderGenotypesRLE::EncodeRLE(const bcf_type& line, container_type& runs,
 
 	// Genotype maps
 	// Map to 0,1,4,5
-	const BYTE*    map = Constants::TOMAHAWK_ALLELE_REDUCED_MAP;
-	if(shift == 2) map = Constants::TOMAHAWK_ALLELE_SELF_MAP;
+	const BYTE*    map = Constants::ALLELE_REDUCED_MAP;
+	if(shift == 2) map = Constants::ALLELE_SELF_MAP;
 
 	//std::cerr << ppa[0] << std::endl;
 	const SBYTE& fmt_type_value1 = *reinterpret_cast<const SBYTE* const>(&line.data[internal_pos+2*ppa[0]]);

src/core/BlockEntry.cpp

@@ -0,0 +1,11 @@
+#include "../support/TypeDefinitions.h"
+#include "../support/helpers.h"
+#include "BlockEntry.h"
+
+namespace Tachyon{
+namespace Core{
+
+
+
+}
+}

src/core/BlockEntry.h

@@ -1,13 +1,9 @@
 #ifndef CORE_BLOCKENTRY_H_
 #define CORE_BLOCKENTRY_H_
 
-#include "base/StreamContainerHeaderController.h"
-#include "base/StreamContainerHeader.h"
-#include "StreamContainer.h"
 #include "PermutationManager.h"
-#include "HashContainer.h"
 #include "../index/IndexBlockEntry.h"
-#include "../io/bcf/BCFReader.h"
+#include "StreamContainer.h"
 
 namespace Tachyon{
 namespace Core{
@@ -161,6 +157,4 @@ class BlockEntry{
 }
 }
 
-
-
 #endif /* CORE_BLOCKENTRY_H_ */

src/core/Importer.cpp

@@ -96,23 +96,25 @@ bool Importer::BuildBCF(void){
 			}
 		}
 
-		std::cerr <<"META-HOT\t" << this->block.meta_hot_container.buffer_data.size() << '\t' << 0 << std::endl;
 		this->block.index_entry.controller.hasGT = 1;
 		this->block.index_entry.controller.isDiploid = 1;
+
+		std::cerr <<"META-HOT\t" << this->block.meta_hot_container.buffer_data.size() << '\t' << 0 << std::endl;
 		std::cerr <<"META-COLD\t" << this->block.meta_cold_container.buffer_data.size() << '\t' << 0 << std::endl;
 		std::cerr <<"GT RLE\t" << this->block.gt_rle_container.buffer_data.size() << '\t' << 0 << std::endl;
 		std::cerr <<"GT SIMPLE\t" << this->block.gt_simple_container.buffer_data.size() << '\t' << 0 << std::endl;
 
-		stream_container test;
-		std::cerr << "outside: " << test.n_additions << std::endl;
-		std::cerr << "other: " << this->block.info_containers[0].n_additions << std::endl;
-
 		// Update head meta
 		this->block.index_entry.n_info_streams = this->info_fields.size();
 		this->block.index_entry.n_format_streams = this->format_fields.size();
 		this->block.index_entry.n_filter_streams = this->filter_fields.size();
 		this->block.index_entry.n_variants = reader.size();
 		this->block.allocateOffsets(this->info_fields.size(), this->format_fields.size(), this->filter_fields.size());
+
+		Index::IndexBlockEntryBase testBase;
+		std::cerr << "external: " << testBase.contigID << std::endl;
+		Index::IndexBlockEntry test;
+		std::cerr << "external: " << test.contigID << std::endl;
 		this->block.index_entry.constructBitVector(Index::IndexBlockEntry::INDEX_INFO,   this->info_fields,   this->info_patterns);
 		this->block.index_entry.constructBitVector(Index::IndexBlockEntry::INDEX_FORMAT, this->format_fields, this->format_patterns);
 		this->block.index_entry.constructBitVector(Index::IndexBlockEntry::INDEX_FILTER, this->filter_fields, this->filter_patterns);

src/core/Importer.h

@@ -1,5 +1,5 @@
-#ifndef TOMAHAWKIMPORTER_H_
-#define TOMAHAWKIMPORTER_H_
+#ifndef CORE_IMPORTER_H_
+#define CORE_IMPORTER_H_
 
 #include "../support/TypeDefinitions.h"
 #include "../support/helpers.h"
@@ -71,7 +71,7 @@ class Importer {
 	totempole_entry_type totempole_entry;  // totempole entry for indexing
 	radix_sorter_type permutator;
 	header_type* header_;     // header
-	encoder_type encoder;   // RLE packer
+	encoder_type encoder;     // RLE packer
 
 	block_type block;
 
@@ -84,6 +84,6 @@ class Importer {
 };
 
 
-} /* namespace Tomahawk */
+} /* namespace Tachyon */
 
-#endif /* TOMAHAWKIMPORTER_H_ */
+#endif /* CORE_IMPORTER_H_ */

src/core/StreamContainer.cpp

@@ -1,3 +1,9 @@
+#include "../support/TypeDefinitions.h"
+#include "../support/helpers.h"
+#include "../third_party/zlib/zconf.h"
+#include "../third_party/zlib/zlib.h"
+#include "../io/BasicBuffer.h"
+#include "../algorithm/OpenHashTable.h"
 #include "StreamContainer.h"
 
 namespace Tachyon{

src/core/StreamContainer.h

@@ -1,14 +1,8 @@
 #ifndef CORE_STREAMCONTAINER_H_
 #define CORE_STREAMCONTAINER_H_
 
-#include "../support/TypeDefinitions.h"
-#include "../support/helpers.h"
-#include "../third_party/zlib/zconf.h"
-#include "../third_party/zlib/zlib.h"
 #include "base/StreamContainerHeaderController.h"
 #include "base/StreamContainerHeader.h"
-#include "../io/BasicBuffer.h"
-#include "../algorithm/OpenHashTable.h"
 
 namespace Tachyon{
 namespace Core{

src/core/base/EntryHotMeta.h

@@ -82,19 +82,22 @@ struct EntryHotMetaBase{
 public:
 	controller_byte controller;
 	U64 position;
-	// most sites are bi-allelic and simple SNVs
+
+	// Most sites are bi-allelic and simple SNVs
 	// sites that are not bi-allelic and not simple
 	// will be encoded in the complex meta section
 	BYTE ref_alt;
+
 	// AF is pre-computed
-	//float MGF;
 	float AF;
+
 	// FILTER map
 	U16 FILTER_map_ID;
 	// INFO map
 	U16 INFO_map_ID;
 	// FORMAT map
 	U16 FORMAT_map_ID;
+
 	// Hot-cold split structure. pointer to cold data
 	// since a pointer cannot be read from a byte
 	// stream as its memory location changes

src/core/base/GTRecords.h

@@ -7,17 +7,17 @@ namespace Core{
 // We CANNOT place phasing template parameter
 // since if we set it to 0 then we have a
 // 0 width bit field which is illegal
-// To solve this we introduce the TomahawkRunNoPhase
+// To solve this we introduce the TachyonRunNoPhase
 // data structure below
 #pragma pack(1)
 template <class T, BYTE missing = 1>
-struct TomahawkRun{
+struct TachyonRun{
 private:
-	typedef TomahawkRun self_type;
+	typedef TachyonRun self_type;
 
 public:
-	TomahawkRun();	// Disallowed ctor
-	~TomahawkRun(); // Disallowed dtor
+	TachyonRun();	// Disallowed ctor
+	~TachyonRun(); // Disallowed dtor
 
 	friend std::ostream& operator<<(std::ostream& out, const self_type& entry){
 		out << (U32)entry.alleleA
@@ -34,13 +34,13 @@ struct TomahawkRun{
 
 #pragma pack(1)
 template <class T, BYTE missing = 1>
-struct TomahawkRunNoPhase{
+struct TachyonRunNoPhase{
 private:
-	typedef TomahawkRunNoPhase self_type;
+	typedef TachyonRunNoPhase self_type;
 
 public:
-	TomahawkRunNoPhase();  // Disallowed ctor
-	~TomahawkRunNoPhase(); // Disallowed dtor
+	TachyonRunNoPhase();  // Disallowed ctor
+	~TachyonRunNoPhase(); // Disallowed dtor
 
 	T alleleA: (1 + missing),
 	  alleleB: (1 + missing),
@@ -49,13 +49,13 @@ struct TomahawkRunNoPhase{
 
 #pragma pack(1)
 template <class T, BYTE missing = 1>
-struct TomahawkRunPacked{
+struct TachyonRunPacked{
 private:
-	typedef TomahawkRunPacked self_type;
+	typedef TachyonRunPacked self_type;
 
 public:
-	TomahawkRunPacked();  // Disallowed ctor
-	~TomahawkRunPacked(); // Disallowed dtor
+	TachyonRunPacked();  // Disallowed ctor
+	~TachyonRunPacked(); // Disallowed dtor
 
 	T phasing: 1,
 	  alleles: 2*(1 + missing),
@@ -64,13 +64,13 @@ struct TomahawkRunPacked{
 
 #pragma pack(1)
 template <class T>
-struct TomahawkRunSimple{
+struct TachyonRunSimple{
 private:
-	typedef TomahawkRunSimple<T> self_type;
+	typedef TachyonRunSimple<T> self_type;
 
 public:
-	TomahawkRunSimple();  // Disallowed ctor
-	~TomahawkRunSimple(); // Disallowed dtor
+	TachyonRunSimple();  // Disallowed ctor
+	~TachyonRunSimple(); // Disallowed dtor
 
 	friend std::ostream& operator<<(std::ostream& out, const self_type& entry){
 		out << (entry.alleleA == 0 ? '.' : (U16)entry.alleleA - 1)

src/core/base/StreamContainerHeader.h

@@ -264,6 +264,4 @@ struct StreamContainerHeaderStride{
 }
 }
 
-
-
 #endif /* CORE_BASE_STREAMCONTAINERHEADER_H_ */

src/import.cpp

@@ -1,6 +1,6 @@
 /*
 Copyright (C) 2016-2017 Genome Research Ltd.
-Author: Marcus D. R. Klarqvist <mk21@sanger.ac.uk>
+Author: Marcus D. R. Klarqvist <mk819@cam.ac.uk>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -114,17 +114,6 @@ int main(int argc, char** argv){
 		std::cerr << Tachyon::Helpers::timestamp("LOG") << "Calling import..." << std::endl;
 	}
 
-	//Tomahawk::TomahawkImportWriter writer;
-	//writer.test();
-	/*
-	Tomahawk::Index::IndexBlockEntryBase b;
-	std::cerr << b.contigID << std::endl;
-	Tomahawk::Index::IndexBlockEntry e;
-	std::cerr << e.contigID << std::endl;
-	return(1);
-	*/
-
-
 	Tachyon::Importer importer(input, output, checkpoint);
 
 	if(!importer.Build())

src/index/IndexBlockEntry.cpp

@@ -60,6 +60,7 @@ void IndexBlockEntry::reset(void){
 }
 
 bool IndexBlockEntry::constructBitVector(const INDEX_BLOCK_TARGET& target, hash_container_type& values, hash_vector_container_type& patterns){
+	std::cerr << "internal bitvector: " << this->contigID << std::endl;
 	if(values.size() == 0)
 		return false;