diff --git a/grim/CONTRIBUTING.md b/grim/CONTRIBUTING.md
new file mode 100755
index 0000000..ab59bd8
--- /dev/null
+++ b/grim/CONTRIBUTING.md
@@ -0,0 +1,58 @@
+Contributing to graph-imputation-match
+===
+
+1) Log into Github web interface with your username-nmdp account
+
+2) Browse to the repo at https://github.com/nmdp-bioinformatics/graph-imputation-match, hit the Fork button
+
+3) Copy the clone URL from the Github web page for the fork (something like https://github.com/username-nmdp/graph-imputation-match.git)
+
+4) Clone the fork
+```bash
+$ git clone https://github.com/username-nmdp/graph-imputation-match.git
+$ cd graph-imputation-match
+```
+
+5) Add upstream as remote
+```bash
+$ git remote add upstream https://github.com/nmdp-bioinformatics/graph-imputation-match
+```
+
+6) Pull and merge latest changes from upstream master to your local master branch
+```bash
+$ git checkout master
+$ git pull upstream master
+$ git push
+```
+
+7) Create a new local feature branch
+```bash
+$ git checkout -b new-feature-branch
+```
+
+8) Edit files locally
+
+9) Commit changes to local feature branch
+```bash
+$ git commit -m "made changes"
+```
+
+10) Push changes from local feature branch to remote feature branch on your fork
+```bash
+$ git push origin new-feature-branch
+```
+
+11) Browse to the Github web page for your fork repo (something like https://github.com/username-nmdp/graph-imputation-match) and hit the new pull request button
+
+12) Edit the pull request description and hit save (or create or new, I can't remember) button
+
+13) Other contributors will review the changes in the pull request
+
+14) When the pull request looks good, it is merged into the upstream repo
+
+15) Hit the delete branch button to delete your remote feature branch (the commits have been merge upstream, so it is no longer necessary)
+
+16) Delete your local feature branch
+```bash
+$ git branch -d new-feature-branch
+```
diff --git a/grim/Dockerfile b/grim/Dockerfile
new file mode 100755
index 0000000..a3b0605
--- /dev/null
+++ b/grim/Dockerfile
@@ -0,0 +1,38 @@
+FROM python:3.8
+
+MAINTAINER "Pradeep Bashyal" pbashyal@nmdp.org
+
+# Setup Repo for Neo4j version 3.5
+RUN wget -O - https://debian.neo4j.com/neotechnology.gpg.key | apt-key add - && \
+ echo 'deb https://debian.neo4j.com stable 3.5' | tee /etc/apt/sources.list.d/neo4j.list
+
+# Install neo4j and cpanm
+RUN apt-get update && \
+ apt-get -y install neo4j cpanminus && \
+ apt-get autoremove
+
+# Install the Perl libraries
+RUN cpanm install --auto-cleanup 0 --notest \
+ REST::Client JSON JSON::Parse Math::Round
+
+# Disable PIP Version check
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+# Disable cache
+ENV PIP_NO_CACHE_DIR=1
+
+# Install Python libraries
+RUN pip install pandas==1.0.3 networkx==2.3 py2neo==4.3.0
+
+# Map cloned grimm directory to /grimm
+# -v $PWD:/grimm
+VOLUME /grimm
+WORKDIR /grimm
+
+# Setup env for Neo4J
+ENV NEO4J_HOME=/usr/share/neo4j
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+
+COPY docker-entrypoint.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+
diff --git a/grim/LICENSE b/grim/LICENSE
new file mode 100755
index 0000000..65c5ca8
--- /dev/null
+++ b/grim/LICENSE
@@ -0,0 +1,165 @@
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/grim/README.md b/grim/README.md
new file mode 100755
index 0000000..df70543
--- /dev/null
+++ b/grim/README.md
@@ -0,0 +1,175 @@
+
+
+# graph-imputation-match
+
+
+
+Graph-based imputation and matching.
+
+| Subdirectory | Description |
+| :-------------- | :---------- |
+| imputation | graph-based imputation |
+| validation | validation of components of this repository |
+| matching | graph-based matching |
+
+# Walkthrough (With Docker)
+
+Make sure you have [Docker](https://www.docker.com/products/docker-desktop) installed. If you're running Docker Desktop,
+you will need to bump up the memory given to Docker. See Docker -> Preferences -> Resources.
+
+A Docker image is available with all the pre-requisite software installed to run the whole pipeline.
+It includes JDK, Neo4j, Perl + Libraries and Python.
+
+Make sure you're in the `graph-imputation-match` directory when running `docker` commands.
+
+You may need to run `sudo docker` depending on how docker was installed.
+
+## Run minimal build
+
+This will build the imputation library and run validation on 20 AAFA simulated sample data.
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx
+```
+
+## Run WMDA Validation
+
+Run imputation and matching on the sample WMDA validation data.
+This requires at least 16Gb of memory. Suggest EC2 size: t2.xlarge
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx wmda
+```
+
+This will produce a `matching/output/cmp.txt` file which lists the difference in the grimm version versus Consensus rsults.
+
+```
+wc -l output/cmp.txt
+ 0 output/cmp.txt
+```
+
+## Run Haplogic Validation
+
+### 6 Loci
+
+This will run the 6-Loci version of validation. (includes DRB4/5/6)
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx haplogic 6loci
+```
+
+### 5 Loci
+
+This will run the 5-Loci version of validation.
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx haplogic 5loci
+```
+
+### Number of parallel processes
+
+By default Haplogic processes will use `n - 1` cpu available. You can specify number of parallel processes by passing in `NUM_OF_PROCESSES` argument.
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx haplogic 5loci NUM_OF_PROCESSES=6
+```
+
+## Clean output directory
+
+Remove the output directories.
+
+```
+docker run --name grimm --rm -v $PWD:/grimm nmdpbioinformatics/grimm-networkx clean
+```
+
+## Manually run inside the container
+
+The following gives you a `bash` shell inside the container where all the pre-req software is installed.
+This mounts the current directory in `/grimm/`
+
+```
+docker run --name grimm --rm -it -v $PWD:/grimm nmdpbioinformatics/grimm-networkx bash
+```
+
+# Walkthrough (Without Docker)
+
+## 0. Pre-requisites
+
+- Clone the repository
+```
+git clone https://github.com/nmdp-bioinformatics/graph-imputation-match
+```
+
+- Python 3
+The project requires Python version 3. Install from If on MacOS python3 can be installed with homebrew as well.
+```
+brew install python3
+```
+
+- Setup Python3 virtual environment
+```
+cd graph-imputation-match
+python3 -m venv venv
+source venv/bin/activate
+```
+# Validate
+`./build-imputation-validation.sh` script runs through graph generation and imputation with minimal configuration.
+
+The configuration file is a json file that's used by graph generation and imputation process to select parameters.
+Supply a new configuration file to test new population set/freq threshold/priors etc.
+
+`conf/minimal-configuration.json` is used for testing that any code changes does not break the build. Make a copy of this file to create your own config.
+
+# Individual Steps
+## 1. Generate Imputation Graph
+ Generate haplotype frequency CSV files.
+
+### Build nemo nodes/csv files
+
+```
+cd imputation
+./build.sh $PWD/../conf/minimal-configuration.json
+cd ..
+ ```
+
+Output directory is created and *edges*, *nodes*, *top links* and *info node* CSV files are generated.
+```
+imputation/graph_generation/output
+├── [ 192] csv
+│ ├── [969M] edges.csv
+│ ├── [ 221] info_node.csv
+│ ├── [ 92M] nodes.csv
+│ └── [ 71M] top_links.csv
+└── [ 25M] hpf.csv
+```
+
+## 2. Perform Imputation
+Perform imputation on simulated data.
+
+```
+cd validation
+```
+
+The input data lives in `data` folder.
+```
+data
+└── [110K] AAFA_NAMER_DONOR_20.in
+```
+
+Start Simulated Imputation
+ ```
+ ./runimputation.sh ../conf/minimal-configuration.json
+ ```
+
+The imputed results are in the `output` directory:
+
+```
+output
+├── [1.9M] AAFA_NAMER_DONOR_20.in_outdeltaFunc_hap
+├── [ 848] AAFA_NAMER_DONOR_20.in_outdeltaFunc_pop
+├── [ 0] AAFA_NAMER_DONOR_20.miss
+└── [ 0] AAFA_NAMER_DONOR_20.problem
+```
+
+## 3. Perform Matching
+ Follow [Match Validation](matching/README.md) to perform matching and compare against consensus results.
diff --git a/grim/build-imputation-validation.sh b/grim/build-imputation-validation.sh
new file mode 100755
index 0000000..889b20b
--- /dev/null
+++ b/grim/build-imputation-validation.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+CONFIG_FILE=${1:-conf/minimal-configuration.json}
+CONFIG_FULL_PATH=${PWD}/${CONFIG_FILE}
+
+echo "Using config file:" ${CONFIG_FULL_PATH}
+
+# Build imputation files
+cd imputation
+./build.sh ${CONFIG_FULL_PATH}
+cd ..
+
+# validation
+cd validation
+./runimputation.sh ${CONFIG_FULL_PATH}
diff --git a/grim/buildspec.yml b/grim/buildspec.yml
new file mode 100755
index 0000000..b56d2d8
--- /dev/null
+++ b/grim/buildspec.yml
@@ -0,0 +1,17 @@
+version: 0.2
+
+env:
+ variables:
+ APP_NAME: "graph-imputation"
+
+
+phases:
+ install:
+ commands:
+ - python --version
+ finally:
+ - echo "Done installing"
+
+ build:
+ commands:
+ - ./build-imputation-validation.sh
diff --git a/grim/conf/README.md b/grim/conf/README.md
new file mode 100755
index 0000000..b486cc1
--- /dev/null
+++ b/grim/conf/README.md
@@ -0,0 +1,19 @@
+##Configuration file parameters:
+
+| Parameter | Description |
+| --- | --- |
+| populations | The population to consider them frequencies. |
+| priority | The coefficient values that define the priority matrix. |
+| loci_map| Loci full name Mapping for indexes. |
+| freq_trim_threshold | The numerator in the frequency threshold. |
+| factor_missing_data | factor to haplotype frequency in plan B in missing data case |
+| Plan_B_Matrix | matrix arranged by the most probable possibilities for recombination. The first element in the matrix should be the full haplotype. the indexes are corresponding to loci_map|
+| planb| True - use plan B anc C. False - use only Plan A. |
+| imputation_in_file | Population genotypes input path. |
+| imputation_out_umug_freq_filename | imputation output. |
+| freq_file | Output file path. |
+| number_of_results | A maximum number of corresponding umug/haplotyes pair for each individual. |
+| epsilon | Genotypes with smaller probability than this value will be discarded. |
+| max_haplotypes_number_in_phase | Limits the number of processed haplotypes in each phase. |
+| Plan_A_Matrix | A list of nodes to plan A. The full-locus nodes will be created anyway, whether they are on the list or not. If the list is empty or this field is missing, all nodes and edges will be created. |
+| save_space_mode | Reduce options in plan B and C if there are too much alleles. Suitable for 9-locus. Default - False |
diff --git a/grim/conf/bart/5loci-configuration.json b/grim/conf/bart/5loci-configuration.json
new file mode 100755
index 0000000..a37361d
--- /dev/null
+++ b/grim/conf/bart/5loci-configuration.json
@@ -0,0 +1,74 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "freq_trim_threshold": 0.5,
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5]],
+ [[1, 2, 3], [4, 5]],
+ [[1], [2, 3], [4, 5]],
+ [[1, 2, 3], [4], [5]],
+ [[1], [2, 3], [4], [5]],
+ [[1], [2], [3], [4], [5]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/bart/data/hla.val.csv.gz",
+ "imputation_out_umug_freq_filename": "bart.umug.freqs",
+ "imputation_out_umug_pops_filename": "bart.umug.pops",
+ "imputation_out_hap_freq_filename": "bart.hap.freqs",
+ "imputation_out_hap_pops_filename": "bart.hap.pops",
+ "imputation_out_miss_filename": "bart.miss",
+ "imputation_out_problem_filename": "bart.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5],[2,3,4], [1,2,3], [1,2,5], [1,2,3,4],[1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt"
+}
diff --git a/grim/conf/bart/9loci-configuration.json b/grim/conf/bart/9loci-configuration.json
new file mode 100755
index 0000000..46e9787
--- /dev/null
+++ b/grim/conf/bart/9loci-configuration.json
@@ -0,0 +1,58 @@
+{
+ "populations": [
+ "AFA",
+ "CAU",
+ "HIS",
+ "API"
+ ],
+ "freq_trim_threshold": 10,
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "loci_map": {
+ "A": 1,
+ "C": 2,
+ "B": 3,
+ "DRBX": 4,
+ "DRB3": 4, "DRB4": 4, "DRB5": 4,
+ "DRB1": 5,
+ "DQA1": 6,
+ "DQB1": 7,
+ "DPA1": 8,
+ "DPB1": 9
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5, 6, 7, 8, 9]],
+ [[1, 2, 3], [4, 5, 6, 7, 8, 9]],
+ [[1, 2, 3], [4, 5, 6, 7] , [8, 9]],
+ [[1, 2, 3], [4, 5], [6, 7], [8, 9]],
+ [[1], [2, 3], [4, 5], [6, 7], [8, 9]],
+ [[1], [2], [3], [4], [5], [6], [7], [8], [9]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-6,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/NEMO2020/9loc/trim/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/bart/data/hla.val.AFA.csv.gz",
+ "imputation_out_umug_freq_filename": "bart.umug.freqs",
+ "imputation_out_umug_pops_filename": "bart.umug.pops",
+ "imputation_out_hap_freq_filename": "bart.hap.freqs",
+ "imputation_out_hap_pops_filename": "bart.hap.pops",
+ "imputation_out_miss_filename": "bart.miss",
+ "imputation_out_problem_filename": "bart.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5],[2,3,4], [1,2,3], [1,2,5], [1,2,3,4],[1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt",
+ "save_space_mode": true
+}
diff --git a/grim/conf/haplogic/5loci-don-configuration.json b/grim/conf/haplogic/5loci-don-configuration.json
new file mode 100755
index 0000000..363fa0b
--- /dev/null
+++ b/grim/conf/haplogic/5loci-don-configuration.json
@@ -0,0 +1,74 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "freq_trim_threshold": 0.5,
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5]],
+ [[1, 2, 3], [4, 5]],
+ [[1], [2, 3], [4, 5]],
+ [[1, 2, 3], [4], [5]],
+ [[1], [2, 3], [4], [5]],
+ [[1], [2], [3], [4], [5]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/5loci_valid_don.gl.csv.gz",
+ "imputation_out_umug_freq_filename": "don.umug.freqs",
+ "imputation_out_umug_pops_filename": "don.umug.pops",
+ "imputation_out_hap_freq_filename": "don.hap.freqs",
+ "imputation_out_hap_pops_filename": "don.hap.pops",
+ "imputation_out_miss_filename": "don.miss",
+ "imputation_out_problem_filename": "don.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5], [2,3,4], [1,2,3], [1,2,5], [1,2,3,4], [1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt"
+}
diff --git a/grim/conf/haplogic/5loci-pat-configuration.json b/grim/conf/haplogic/5loci-pat-configuration.json
new file mode 100755
index 0000000..3410fc5
--- /dev/null
+++ b/grim/conf/haplogic/5loci-pat-configuration.json
@@ -0,0 +1,74 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "freq_trim_threshold": 0.5,
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5]],
+ [[1, 2, 3], [4, 5]],
+ [[1], [2, 3], [4, 5]],
+ [[1, 2, 3], [4], [5]],
+ [[1], [2, 3], [4], [5]],
+ [[1], [2], [3], [4], [5]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/5loci_valid_pat.gl.csv.gz",
+ "imputation_out_umug_freq_filename": "pat.umug.freqs",
+ "imputation_out_umug_pops_filename": "pat.umug.pops",
+ "imputation_out_hap_freq_filename": "pat.hap.freqs",
+ "imputation_out_hap_pops_filename": "pat.hap.pops",
+ "imputation_out_miss_filename": "pat.miss",
+ "imputation_out_problem_filename": "pat.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5], [2,3,4], [1,2,3], [1,2,5], [1,2,3,4], [1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt"
+}
diff --git a/grim/conf/haplogic/6loci-don-configuration.json b/grim/conf/haplogic/6loci-don-configuration.json
new file mode 100755
index 0000000..929fac8
--- /dev/null
+++ b/grim/conf/haplogic/6loci-don-configuration.json
@@ -0,0 +1,78 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "freq_trim_threshold": 0.5,
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5,
+ "DRBX": 6,
+ "DRB3": 6,
+ "DRB4": 6,
+ "DRB5": 6
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014_6Locus/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/6loci_valid_don.gl.csv.gz",
+ "imputation_out_umug_freq_filename": "don.umug.freqs",
+ "imputation_out_umug_pops_filename": "don.umug.pops",
+ "imputation_out_hap_freq_filename": "don.hap.freqs",
+ "imputation_out_hap_pops_filename": "don.hap.pops",
+ "imputation_out_miss_filename": "don.miss",
+ "imputation_out_problem_filename": "don.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5], [2,3,4], [1,2,3], [1,2,5], [1,2,3,4], [1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt"
+}
diff --git a/grim/conf/haplogic/6loci-pat-configuration.json b/grim/conf/haplogic/6loci-pat-configuration.json
new file mode 100755
index 0000000..b27802f
--- /dev/null
+++ b/grim/conf/haplogic/6loci-pat-configuration.json
@@ -0,0 +1,78 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "freq_trim_threshold": 0.5,
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5,
+ "DRBX": 6,
+ "DRB3": 6,
+ "DRB4": 6,
+ "DRB5": 6
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 1000,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014_6Locus/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/6loci_valid_pat.gl.csv.gz",
+ "imputation_out_umug_freq_filename": "pat.umug.freqs",
+ "imputation_out_umug_pops_filename": "pat.umug.pops",
+ "imputation_out_hap_freq_filename": "pat.hap.freqs",
+ "imputation_out_hap_pops_filename": "pat.hap.pops",
+ "imputation_out_miss_filename": "pat.miss",
+ "imputation_out_problem_filename": "pat.problem",
+ "max_haplotypes_number_in_phase": 100,
+ "Plan_A_Matrix": [[1,2,3,4,5], [2,3,4], [1,2,3], [1,2,5], [1,2,3,4], [1,2,4,5]],
+ "pops_count_file": "imputation/graph_generation/output/pop_ratio.txt"
+}
diff --git a/grim/conf/haplogic/don-100-configuration.json b/grim/conf/haplogic/don-100-configuration.json
new file mode 100755
index 0000000..667180e
--- /dev/null
+++ b/grim/conf/haplogic/don-100-configuration.json
@@ -0,0 +1,78 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "freq_trim_threshold": 1e-3,
+ "priority": {
+ "alpha": 0,
+ "eta": 0,
+ "beta": 0,
+ "gamma": 0,
+ "delta": 1
+ },
+ "FULL_LOCI": "ABCQRX",
+ "loci_map": {
+ "A": "A",
+ "B": "B",
+ "C": "C",
+ "DQB1": "Q",
+ "DRB1": "R",
+ "DRBX": "X",
+ "DRB3": "X",
+ "DRB4": "X",
+ "DRB5": "X"
+ },
+ "factor_missing_data": 0.0001,
+ "matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "planb": true,
+ "output_MUUG": true,
+ "output_haplotypes": false,
+ "epsilon": 1e-5,
+ "number_of_results": 100,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014_6Locus/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/sample/6loci_don_100.txt.gz",
+ "imputation_out_umug_freq_filename": "don.umug.freqs",
+ "imputation_out_umug_pops_filename": "don.umug.pops",
+ "imputation_out_hap_freq_filename": "don.hap.freqs",
+ "imputation_out_hap_pops_filename": "don.hap.pops",
+ "imputation_out_miss_filename": "don.miss",
+ "imputation_out_problem_filename": "don.problem",
+ "max_haplotypes_number_in_phase": 100
+}
diff --git a/grim/conf/haplogic/haplogic-minimal-configuration.json b/grim/conf/haplogic/haplogic-minimal-configuration.json
new file mode 100755
index 0000000..b5e7e53
--- /dev/null
+++ b/grim/conf/haplogic/haplogic-minimal-configuration.json
@@ -0,0 +1,50 @@
+{
+ "populations": [
+ "AAFA",
+ "CARB",
+ "FILII",
+ "MENAFC",
+ "NAMER"
+ ],
+ "priority": {
+ "alpha": 0,
+ "eta": 1,
+ "beta": 0,
+ "gamma": 0,
+ "deltaFunc": false
+ },
+ "FULL_LOCI": "ABCQRX",
+ "loci_map": {
+ "A": "A",
+ "B": "B",
+ "C": "C",
+ "DQB1": "Q",
+ "DRB1": "R",
+ "DRBX": "X",
+ "DRB3": "X",
+ "DRB4": "X",
+ "DRB5": "X"
+ },
+ "factor_missing_data": 0.0001,
+ "matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "freq_trim_threshold": 1e-5,
+ "freq_data_dir": "imputation/graph_generation/data/NEMO2011/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/pat_100.txt.gz",
+ "imputation_out_umug_freq_filename": "pat.umug.freqs",
+ "imputation_out_umug_pops_filename": "pat.umug.pops",
+ "imputation_out_hap_freq_filename": "pat.hap.freqs",
+ "imputation_out_hap_pops_filename": "pat.hap.pops",
+ "imputation_out_miss_filename": "pat.miss",
+ "imputation_out_problem_filename": "pat.problem"
+}
diff --git a/grim/conf/haplogic/haplogic-validation-configuration.json b/grim/conf/haplogic/haplogic-validation-configuration.json
new file mode 100755
index 0000000..c5f76f5
--- /dev/null
+++ b/grim/conf/haplogic/haplogic-validation-configuration.json
@@ -0,0 +1,75 @@
+{
+ "planb": false,
+ "epsilon": 1e-9,
+ "number_of_results": 20,
+ "number_of_pop_results": 20,
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "priority": {
+ "alpha": 1,
+ "eta": 0,
+ "beta": 0,
+ "gamma": 0,
+ "deltaFunc": false
+ },
+ "FULL_LOCI": "ABCQRX",
+ "loci_map": {
+ "A": "A",
+ "B": "B",
+ "C": "C",
+ "DQB1": "Q",
+ "DRB1": "R",
+ "DRBX": "X",
+ "DRB3": "X",
+ "DRB4": "X",
+ "DRB5": "X"
+ },
+ "factor_missing_data": 0.0001,
+ "matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "freq_trim_threshold": 1e-9,
+ "freq_data_dir": "imputation/graph_generation/data/NEMO2011/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/don.gl.mr.txt.gz",
+ "imputation_out_umug_freq_filename": "don.umug.freqs",
+ "imputation_out_umug_pops_filename": "don.umug.pops",
+ "imputation_out_hap_freq_filename": "don.hap.freqs",
+ "imputation_out_hap_pops_filename": "don.hap.pops",
+ "imputation_out_miss_filename": "don.miss",
+ "imputation_out_problem_filename": "don.problem"
+}
diff --git a/grim/conf/haplogic/pat-100-configuration.json b/grim/conf/haplogic/pat-100-configuration.json
new file mode 100755
index 0000000..527a551
--- /dev/null
+++ b/grim/conf/haplogic/pat-100-configuration.json
@@ -0,0 +1,78 @@
+{
+ "populations": [
+ "AAFA",
+ "AFB",
+ "AINDI",
+ "AISC",
+ "ALANAM",
+ "AMIND",
+ "CARB",
+ "CARHIS",
+ "CARIBI",
+ "HAWI",
+ "FILII",
+ "KORI",
+ "JAPI",
+ "MSWHIS",
+ "MENAFC",
+ "NAMER",
+ "NCHI",
+ "SCAHIS",
+ "SCAMB",
+ "SCSEAI",
+ "VIET",
+ "AFA",
+ "API",
+ "CAU",
+ "HIS",
+ "NAM"
+ ],
+ "freq_trim_threshold": 1e-3,
+ "priority": {
+ "alpha": 0,
+ "eta": 0,
+ "beta": 0,
+ "gamma": 0,
+ "delta": 1
+ },
+ "FULL_LOCI": "ABCQRX",
+ "loci_map": {
+ "A": "A",
+ "B": "B",
+ "C": "C",
+ "DQB1": "Q",
+ "DRB1": "R",
+ "DRBX": "X",
+ "DRB3": "X",
+ "DRB4": "X",
+ "DRB5": "X"
+ },
+ "factor_missing_data": 0.0001,
+ "matrix": [
+ [[1, 2, 3, 4, 5, 6]],
+ [[1, 2, 3], [4, 5, 6]],
+ [[1], [2, 3], [4, 5, 6]],
+ [[1, 2, 3], [4], [5], [6]],
+ [[1], [2, 3], [4], [5], [6]],
+ [[1], [2], [3], [4], [5], [6]]
+ ],
+ "planb": true,
+ "output_MUUG": true,
+ "output_haplotypes": false,
+ "epsilon": 1e-5,
+ "number_of_results": 100,
+ "number_of_pop_results": 100,
+ "freq_data_dir": "imputation/graph_generation/data/2014_6Locus/",
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/haplogic/data/sample/6loci_pat_100.txt.gz",
+ "imputation_out_umug_freq_filename": "pat.umug.freqs",
+ "imputation_out_umug_pops_filename": "pat.umug.pops",
+ "imputation_out_hap_freq_filename": "pat.hap.freqs",
+ "imputation_out_hap_pops_filename": "pat.hap.pops",
+ "imputation_out_miss_filename": "pat.miss",
+ "imputation_out_problem_filename": "pat.problem",
+ "max_haplotypes_number_in_phase": 100
+}
diff --git a/grim/conf/minimal-configuration.json b/grim/conf/minimal-configuration.json
new file mode 100644
index 0000000..e1de8aa
--- /dev/null
+++ b/grim/conf/minimal-configuration.json
@@ -0,0 +1,50 @@
+{
+ "populations": [
+ "FILII",
+ "NAMER"
+ ],
+ "freq_trim_threshold": 1e-5,
+ "priority": {
+ "alpha": 0.4999999,
+ "eta": 0,
+ "beta": 1e-7,
+ "gamma": 1e-7,
+ "delta": 0.4999999
+ },
+ "FULL_LOCI": "ABCQR",
+ "loci_map": {
+ "A": 1,
+ "B": 2,
+ "C": 3,
+ "DQB1": 4,
+ "DRB1": 5
+ },
+ "factor_missing_data": 0.0001,
+ "Plan_B_Matrix": [
+ [[1, 2, 3, 4, 5]],
+ [[1, 2, 3], [4, 5]],
+ [[1], [2, 3], [4, 5]],
+ [[1, 2, 3], [4], [5]],
+ [[1], [2, 3], [4], [5]],
+ [[1], [2], [3], [4], [5]]
+ ],
+ "planb": true,
+ "number_of_options_threshold": 100000,
+ "epsilon": 1e-3,
+ "number_of_results": 10,
+ "number_of_pop_results": 100,
+ "output_MUUG": true,
+ "output_haplotypes": true,
+ "node_csv_file": "imputation/graph_generation/output/csv/nodes.csv",
+ "edges_csv_file": "imputation/graph_generation/output/csv/edges.csv",
+ "info_node_csv_file": "imputation/graph_generation/output/csv/info_node.csv",
+ "top_links_csv_file": "imputation/graph_generation/output/csv/top_links.csv",
+ "imputation_in_file": "validation/simulation/data/simulated_donor.csv",
+ "imputation_out_umug_freq_filename": "don.umug.freqs",
+ "imputation_out_umug_pops_filename": "don.umug.pops",
+ "imputation_out_hap_freq_filename": "don.hap.freqs",
+ "imputation_out_hap_pops_filename": "don.hap.pops",
+ "imputation_out_miss_filename": "don.miss",
+ "imputation_out_problem_filename": "don.problem",
+ "max_haplotypes_number_in_phase": 100
+}
diff --git a/grim/docker-entrypoint.sh b/grim/docker-entrypoint.sh
new file mode 100755
index 0000000..dae3bf2
--- /dev/null
+++ b/grim/docker-entrypoint.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+CMD=${1:-minimal}
+
+cd /grimm/
+
+if [[ "${CMD}" = "minimal" ]]; then
+ echo "Running minimal build"
+ ./build-imputation-validation.sh
+elif [[ "${CMD}" = "clean" ]]; then
+ echo "Clean output directories"
+ cd validation/haplogic
+ make clean_all
+elif [[ "${CMD}" = "wmda" ]]; then
+ echo "Running wmda validation"
+ cd validation/wmda
+ ./wmda-pipeline.sh
+elif [[ "${CMD}" = "haplogic" ]]; then
+ echo "**Running Haplogic Validation**"
+ shift # remove haplogic
+ SUB_CMD=$1
+ shift # remove 5loci/6loci
+ if [[ "${SUB_CMD}" = "6loci" ]]; then
+ echo "6 Loci validation"
+ cd validation/haplogic
+ make PAT_CONFIG_FILE=$PWD/../../conf/haplogic/6loci-pat-configuration.json\
+ DON_CONFIG_FILE=$PWD/../../conf/haplogic/6loci-don-configuration.json \
+ "$@"
+ elif [[ "${SUB_CMD}" = "5loci" ]]; then
+ echo "5 Loci validation"
+ cd validation/haplogic
+ make PAT_CONFIG_FILE=$PWD/../../conf/haplogic/5loci-pat-configuration.json\
+ DON_CONFIG_FILE=$PWD/../../conf/haplogic/5loci-don-configuration.json \
+ "$@"
+ else
+ echo "Unknown option:" "$CMD" "$SUB_CMD" "$@"
+ fi
+elif [[ "${CMD}" = "make" ]]; then
+ echo "Running Haplogic validation"
+ cd validation/haplogic
+ exec "$@"
+else
+ exec "$@"
+fi
+
diff --git a/grim/grim.py b/grim/grim.py
index e3f5479..a0e6324 100644
--- a/grim/grim.py
+++ b/grim/grim.py
@@ -21,19 +21,25 @@
# > http://www.fsf.org/licensing/licenses/lgpl.html
# > http://www.opensource.org/licenses/lgpl-license.php
#
-from .algorithm.match import slug_match
-from .model.slug import SLUG
-
-
-def match(patient_slug_glstring, donor_slug_glstring):
- """
- Perform match on GL String versions of SLUGs
-
- @param patient_slug_glstring: Patient's SLUG in GL String format
- @param donor_slug_glstring: Donor's SLUG in GL String format
- @return: bool indicating Match or No Match
- """
- patient_slug = SLUG.from_glstring(patient_slug_glstring)
- donor_slug = SLUG.from_glstring(donor_slug_glstring)
- matched = slug_match(patient_slug, donor_slug)
- return donor_slug, matched, patient_slug
+
+
+from validation import runfile
+from imputation.graph_generation import generate_neo4j_multi_hpf
+
+from imputation.imputegl import Imputation
+from imputation.imputegl.networkx_graph import Graph
+
+def graph_freqs(conf_file = "./conf/minimal-configuration.json", for_em = False, em_pop=None, freq_file='hpf.csv', path='imputation/graph_generation/' ):
+ generate_neo4j_multi_hpf.generate_graph(freq_file=freq_file, path=path, config_file=conf_file, em_pop=em_pop,
+ em=for_em)
+
+def impute(conf_file = "./conf/minimal-configuration.json"):
+ runfile.run_impute(conf_file)
+
+def impute_instance(config):
+ graph = Graph(config)
+ graph.build_graph(config["node_file"], config["top_links_file"], config["edges_file"])
+ imputation = Imputation(graph, config)
+ return imputation
+
+
diff --git a/grim/imputation/README.md b/grim/imputation/README.md
new file mode 100755
index 0000000..cb9c6bb
--- /dev/null
+++ b/grim/imputation/README.md
@@ -0,0 +1,8 @@
+# NetworkX based imputation
+
+| Subdirectory | Description |
+| :-------------- | :---------- |
+| graph_generation | generate the input CSV files graph from haplotype frequencies |
+| imputegl | perform imputation based on gl-string formatted input |
+
+
diff --git a/grim/imputation/README.rst b/grim/imputation/README.rst
new file mode 100755
index 0000000..e69de29
diff --git a/grim/imputation/build.sh b/grim/imputation/build.sh
new file mode 100755
index 0000000..2a2a674
--- /dev/null
+++ b/grim/imputation/build.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+if [[ -z "$1" ]]
+then
+ echo "Usage: `basename $0` "
+ exit 1
+fi
+
+# Install package
+pip uninstall -y imputegl
+pip uninstall -y imputegl
+
+python setup.py install
+pip install .
+
+# Build graph
+cd graph_generation
+
+make nemo CONFIG_FILE=${1}
+echo "Graph data:"
+ls -lah output/*
+
+cd ..
diff --git a/grim/imputation/graph_generation/LICENSE_INFO.rst b/grim/imputation/graph_generation/LICENSE_INFO.rst
new file mode 100755
index 0000000..82f82d9
--- /dev/null
+++ b/grim/imputation/graph_generation/LICENSE_INFO.rst
@@ -0,0 +1,15 @@
+The WMDA data is part of the supplemental materials from the following
+article.
+
+Bochtler W., Gragert L., Patel Z.I., Robinson J., Steiner D., Hofmann J.A.,
+Pingel J., Baouz A., Melis A., Schneider J., Eberhard H.P., Oudshoorn M.,
+Marsh S.G., Maiers M. & Müller C.R. (2016).
+A comparative reference study for the validation of HLA‐matching algorithms
+in the search for allogeneic hematopoietic stem cell donors and cord blood
+units.
+*HLA*. 2016 Jun;87(6):439-48. doi: 10.1111/tan.12817.
+
+It is distributed under a Creative Commons Attribution‐NonCommercial‐NoDerivs
+(CC BY-NC-ND 4.0) license.
+
+https://creativecommons.org/licenses/by-nc-nd/4.0/
diff --git a/grim/imputation/graph_generation/Makefile b/grim/imputation/graph_generation/Makefile
new file mode 100755
index 0000000..5293ef4
--- /dev/null
+++ b/grim/imputation/graph_generation/Makefile
@@ -0,0 +1,19 @@
+CONFIG_FILE = "../../minimal-configuration.json"
+wmda:
+ rm -rf output
+ python wmda_download.py
+ python wmda_to_hpf_csv.py
+ python generate_neo4j_multi_hpf.py -c $(CONFIG_FILE)
+
+
+nemo:
+ rm -rf output
+ mkdir -p output
+ python nemo_to_hpf_csv.py -c $(CONFIG_FILE)
+ python generate_neo4j_multi_hpf.py -c $(CONFIG_FILE)
+
+em2014:
+ rm -rf output
+ mkdir -p output
+ python nemo_to_hpf_csv.py -c $(CONFIG_FILE)
+ python generate_neo4j_multi_hpf.py -c $(CONFIG_FILE)
diff --git a/grim/imputation/graph_generation/README.bug b/grim/imputation/graph_generation/README.bug
new file mode 100755
index 0000000..0d34432
--- /dev/null
+++ b/grim/imputation/graph_generation/README.bug
@@ -0,0 +1,24 @@
+$ cut -f1,2 -d',' output/csv/edges.csv |sort |uniq -c |sort -rn |more
+ 1 :START_ID(HAPLOTYPE),:END_ID(HAPLOTYPE)
+ 1 9999,813
+ 1 9998,812
+ 1 9997,811
+ 1 9996,810
+ 1 9995,882
+ 1 9995,809
+ 1 9994,808
+ 1 9993,807
+ 1 9992,805
+ 1 9991,804
+
+$ cut -f1,2 -d',' output/csv/nemo/edges.csv |sort |uniq -c |sort -rn |more
+ 753 117906,117357
+ 753 117884,117357
+ 717 117907,117358
+ 717 117883,117358
+ 614 117837,111923
+ 614 117651,111923
+ 539 117913,117365
+ 539 117884,117365
+ 515 117918,117370
+
diff --git a/grim/imputation/graph_generation/README.md b/grim/imputation/graph_generation/README.md
new file mode 100755
index 0000000..3b1362d
--- /dev/null
+++ b/grim/imputation/graph_generation/README.md
@@ -0,0 +1,123 @@
+# Graph Data Generation
+
+## Pre-requisites
+
+### Mac OS
+
+- JDK 8
+ - Install JDK 1.8 from Oracle
+ - Set JAVA_HOME
+ On MacOS, add the follwoing to .bash_profile:
+ ```
+ export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
+ ```
+
+- Python 3
+ - On MacOS install with
+ ```
+ brew install python3
+ ```
+
+- Install Neo4J
+ - On MacOS install with
+ ```
+ brew install neo4j
+ ```
+
+ - Setup NEO4J_HOME
+
+ Point NEO4J_HOME to the root of the NEO4J directory.
+ ```
+ export NEO4J_HOME=/usr/local/Cellar/neo4j/3.2.2/libexec
+ ```
+
+### Linux
+- JDK 8
+ - Install JDK 1.8 from Oracle
+ - add JAVA_HOME to ~/.bash_profile
+ ```
+ export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/bin/java::")
+ ```
+
+
+- Install Neo4J
+ - Download latest compressed version of Neo4j, for example
+ ```
+ wget https://neo4j.com/artifact.php?name=neo4j-community-3.5.7-unix.tar.gz
+ ```
+ - uncompress tar file
+ ```
+ tar -xf artifact.php?name=neo4j-community-3.5.7-unix.tar.gz
+ ```
+
+ - Point NEO4J_HOME to the root of the uncompressed NEO4J directory and add the following line to ~/.bash_profile
+
+ ```
+ export NEO4J_HOME=path/to/neo4j-community-3.5.7
+ ```
+
+
+
+# Using Makefile
+
+For WMDA
+```
+make wmda
+```
+
+For Nemo
+```
+make nemo
+```
+
+# Manually Step by Step
+
+# WMDA Dataset
+- Download and prepare wmda data. Python script downloads reference wmda data and untars it in wmda directory
+```
+ cd graph_generation
+ python wmda_download.py
+```
+
+- Generate nodes/edges/toplinks from the reference wmda data. The freqs file is cnverted to HPF format first.
+```
+ python wmda_to_hpf_csv.py
+ python generate_neo4j_single_hpf.py
+```
+
+- Generated nodes and edges (creates edges; and nodes with “Frequency” float attribute, not array) files are in the output/csv directory.
+
+```
+ output
+ └── csv
+ ├── edges.csv
+ ├── nodes.csv
+ └── top_links.csv
+```
+
+# NEMO Dataset
+
+To use a different set of frequencies use the following procedure:
+
+- Starting in the graph generator directory, convert the data from frequency format to hpf (haplotype, population, frequency).
+```
+ python nemo_to_hpf_csv.py
+```
+
+- This program looks for a data/NEMO2011 directory and reads the individual frequency files and generates this csv:
+```
+ output/hpf.csv
+```
+- To generate the graph files run this script
+```
+ python generate_neo4j_multi_hpf.py
+```
+ Note: the script reads from data/NEMO2011/hpf.csv and writes the three graph csv files to the directory
+```
+ output/csv
+ ├── edges.csv
+ ├── nodes.csv
+ └── top_links.csv
+```
+ Note: there is an option to trim the frequency set below a frequency threshold. If the trimming threshold is 1e-6 it will take 9m35s to generate the graph csv files on a mid-2015 MacBook Pro (2.5 GHz Intel Core i7) and will result in 1,088,817 nodes (159MB), 14,868,976 edges (2.0GB)and 5,947,591 top links (108MB).
+
diff --git a/grim/imputation/graph_generation/README.old b/grim/imputation/graph_generation/README.old
new file mode 100755
index 0000000..052ab1a
--- /dev/null
+++ b/grim/imputation/graph_generation/README.old
@@ -0,0 +1,17 @@
+
+there are 6 python scripts in graph_generator
+
+wmda_to_hpf_csv.py - convert wmda frequencies to hpf (GRIMM)
+generate_neo4j_wmda_hpf.py - genreate graph csvs (GRIMM)
+nemo_to_hpf_csv.py - convert nemo frequencies to hpf
+generate_Neo4J_multi.py - create mutli graph from frequencies files
+** generate_neo4j_hpf.py - same but reads hpf file
+generate_neo4J_load_files.py - early code based on jupyter notebook
+
+
+
+I used this one (**) as the single method to rule them all.
+It generate multiple CP links between nodes (with different values).
+This makes the graph traversal ambiguous.
+Not good.
+
diff --git a/grim/imputation/graph_generation/generate_neo4j_multi_hpf.py b/grim/imputation/graph_generation/generate_neo4j_multi_hpf.py
new file mode 100644
index 0000000..23b0dc2
--- /dev/null
+++ b/grim/imputation/graph_generation/generate_neo4j_multi_hpf.py
@@ -0,0 +1,418 @@
+#
+# read frequency file in hpf format and create 3 csv files for the graph
+#
+import argparse
+import csv
+import json
+import pathlib
+from collections import defaultdict, namedtuple
+from itertools import combinations, count
+from operator import indexOf, add
+
+import sys
+import os
+#sys.path.insert(0, os.path.join(".."))
+
+
+# set haplotype frequency trimming threshold
+#
+# The NEMO frequency generating sample is 6.93M and the largest single
+# cohort is CAU 3,912,440 so a haplotype with a frequency of < 1/7,824,880
+# would have an expected value < 1 in all populations
+# freq_trim = 1e-5 # 0.00001 1 in 100K
+# freq_trim = 1.1278e-7 # 0.0000001278... 1 in 7.8M
+# freq_trim = 1.66e-7 # 0.000000166... 1 in 6M
+# freq_trim = 1e-6 # 0.000001 1 in 1M
+# freq_trim = 1.66e-07
+
+
+# If the trimming threshold is 1e-6 it will take 9m35s to generate the graph
+# csv files on a mid-2015 MacBook Pro (2.5 GHz Intel Core i7) and will result
+# in 1,088,817 nodes (159MB),
+# 14,868,976 edges (2.0GB)and 5,947,591 top links (108MB).
+
+# If the trimming threshold is 1e-7 it will take 19m37 to generate the graph
+# csv files on a mid-2015 MacBook Pro (2.5 GHz Intel Core i7) and will result
+# in 1.8M nodes (262MB),
+# 26M edges (3.6B)and 107M top links (200MB).
+#FULL_LOCI = 'ABCQR'
+
+##############################################################################
+# functions
+##############################################################################
+# nCr: n Choose r
+def nCr(nchars, r):
+ return [''.join(x) for x in combinations(nchars, r)]
+
+
+# dividebyzero: safe division
+def dividebyzero(a, b):
+ if b == 0:
+ return 0
+ else:
+ return a / b
+
+
+# make_allele_list: convert haplotype into sorted list of alleles
+def make_allele_list(haplotype, full_name_index_dict, num_of_alleles):
+ hl = haplotype.split('~')
+ # Remove g from the allele names
+ hl = list(map(lambda h: h[:-1] if (h[-1] == 'g') else h, hl))
+ sorted_h1 = ['0']*num_of_alleles
+ for allele in hl:
+ locus = allele.split('*')[0]
+ sorted_h1[full_name_index_dict[locus] - 1 ] = allele
+ return sorted_h1
+
+
+
+
+
+# Find the index of a loci name in `FULL_LOCI`. ie. `BC` in `ABCRQ` is `[1,2]`
+def find_indices(FULL_LOCI, loci):
+ return list(map(lambda x: indexOf(FULL_LOCI, x), loci))
+
+
+# Given a list of indices find the corresponding loci combo
+def find_loci_name(FULL_LOCI, loci_indices):
+ return ''.join([FULL_LOCI[i] for i in sorted(loci_indices)])
+
+
+# Find all the parents of the loci with loci_indices against loci_allele_list.
+def find_parents(FULL_LOCI, loci_indices, loci_allele_list, p, ParentLoci, top_nodes_plan_b):
+ fi = find_indices(FULL_LOCI, FULL_LOCI)
+ new_loci = set(fi).difference(loci_indices)
+ parents = []
+ for i in list(new_loci):
+ new_loci = loci_indices.copy()
+ new_loci.append(i)
+ # print(new_loci)
+ loci_name = find_loci_name(FULL_LOCI, new_loci)
+ if loci_name in top_nodes_plan_b:
+ # print(loci_name)
+ haplotype = '~'.join([loci_allele_list[i] for i in sorted(new_loci)])
+ parents.append(ParentLoci(loci=loci_name, haplotype=haplotype, p=p))
+ return parents
+
+# Find the wanted nodes for plan A and plan B
+def labels_for_grap(conf, full_loci,csvdir):
+
+ nodes_graph = conf.get("Plan_A_Matrix", [])
+
+ # Created all the node combinations if there is no limit on the requested nodes
+ if not nodes_graph:
+ list_all_combo = [full_loci]
+ for i in range(len(full_loci) - 1, 0, -1):
+ list_all_combo.extend(nCr(full_loci, i))
+ return list_all_combo, list_all_combo, list_all_combo, list_all_combo
+
+ nodes_plan_a = []
+ nodes_plan_b = []# nodes from which edges to the parents came out in plan b
+ top_nodes_plan_b = []# nodes to which edges will reach in plan b or single locus without requested parents.
+
+
+
+ matrix_plan_b = conf.get("Plan_B_Matrix", [
+ [[1, 2, 3, 4, 5]],
+ [[1, 2, 3], [4, 5]],
+ [[1], [2, 3], [4, 5]],
+ [[1, 2, 3], [4], [5]],
+ [[1], [2, 3], [4], [5]],
+ [[1], [2], [3], [4], [5]]
+ ])[1]
+ # convert the first recombination event from loci indexes to loci names,
+ # and add the event subcomponents to top_nodes_plan_b
+ for subcomponent in matrix_plan_b:
+ for idx, locus in enumerate(subcomponent):
+ subcomponent[idx] = str(locus)
+ top_nodes_plan_b.append(('').join(subcomponent))
+
+ list_complement = []
+ full_loci_list = []
+ full_loci_list[:0] = full_loci
+ for node_label in nodes_graph:
+ # add nodes of plan a
+ label = ''
+ list_label = []
+ for locus_idx in node_label:
+ label += str(locus_idx)
+ list_label.append(str(locus_idx))
+
+ nodes_plan_a.append(label)
+ complement_node = ('').join(sorted([item for item in full_loci_list if item not in list_label]))
+ if complement_node != '':
+ list_complement.append(complement_node)
+ # find the sub-nodes of label for plan b
+ for subcomponent in matrix_plan_b:
+ node = ''
+ for locus in label:
+ if locus in subcomponent:
+ node += locus
+ if node != '' and not node in top_nodes_plan_b:
+ nodes_plan_b.append(node)
+
+ #add the full-locus to plan a nodes
+ if not full_loci in nodes_plan_a:
+ nodes_plan_a.append(full_loci)
+
+ # add the single locus to top_nodes_plan_b
+ all_loci = list(full_loci)
+ for locus in all_loci:
+ if not locus in nodes_plan_b:
+ top_nodes_plan_b.append(locus)
+
+ # add the complementary nodes if they are not in one of the graphs
+ for node in list_complement:
+ if (not node in nodes_plan_a) and (not node in nodes_plan_b) and (not node in top_nodes_plan_b):
+ top_nodes_plan_b.append(node)
+
+ nodes_plan_b = list(set(nodes_plan_b))
+ all_combo_list = list(dict.fromkeys(nodes_plan_a + nodes_plan_b + top_nodes_plan_b))
+ with open(csvdir + '/nodes_for_plan_a.txt', 'w') as f:
+ for item in nodes_plan_a:
+ f.write("%s\n" % item)
+ with open(csvdir + '/nodes_for_plan_b.txt', 'w') as f:
+ for item in (nodes_plan_b + top_nodes_plan_b):
+ f.write("%s\n" % item)
+ #pickle.dump(nodes_plan_a, open(csvdir + '/nodes_for_plan_a.pkl', "wb"))
+ #pickle.dump(nodes_plan_b + top_nodes_plan_b, open(csvdir + '/nodes_for_plan_b.pkl', "wb"))
+
+ return all_combo_list, nodes_plan_a, nodes_plan_b, top_nodes_plan_b
+
+#create full loci order,
+#map beteen the loci neme and his index
+def loci_order(loc_values):
+ full_name_index_dict = {}
+ all_loci_set = set()
+ for locus, val in loc_values.items():
+ full_name_index_dict[locus] = val
+ all_loci_set.add(str(val))
+
+ FULL_LOCI = ''.join( sorted(all_loci_set))
+
+ return FULL_LOCI, full_name_index_dict
+
+def generate_graph(freq_file = 'hpf.csv', path = '', config_file = "../../conf/minimal-configuration.json", em_pop = None, em= False):
+ ##############################################################################
+ # Configure
+ ##############################################################################
+ # set output directory and create it if it doesn't exist
+ csvdir = path +"output/csv"
+ pathlib.Path(csvdir).mkdir(parents=True, exist_ok=True)
+
+ # Input file
+ freq_file = path + 'output/' + freq_file
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-c", "--config",
+ required=False,
+ default= config_file,
+ help="Configuration JSON file",
+ type=str)
+
+ args = parser.parse_args()
+ configuration_file = args.config
+
+ # Read configuration file and load properties
+ with open(configuration_file) as f:
+ conf = json.load(f)
+
+ pops = conf.get("populations")
+ if em_pop:
+ pops = em_pop
+ freq_trim = conf.get("freq_trim_threshold")
+
+ dict_count_of_pop = {}
+ #if em:
+ for pop in pops:
+ dict_count_of_pop[pop] = freq_trim
+ """else:
+ project_dir = "../../"
+ pop_ratio_dir = project_dir + conf.get("pops_count_file", 'imputation/graph_generation/output/pop_ratio.txt')
+ with open(pop_ratio_dir) as f_count:
+ for line in f_count:
+ pop, count_pop, ratio = line.strip().split(',')
+ dict_count_of_pop[pop] = freq_trim / float(count_pop)"""
+
+
+ # Display the configurations we are using
+ print('****************************************************************************************************')
+ print("Performing graph generation based on following configuration:")
+ print("\tPopulation: {}".format(pops))
+ print("\tFreq File: {}".format(freq_file))
+ print("\tFreq Trim Threshold: {}".format(freq_trim))
+ print('****************************************************************************************************')
+
+ ##############################################################################
+ # setup graph elements
+ ##############################################################################
+ HaplotypeNode = namedtuple('HaplotypeNode', 'node_id, freq_list, allele_list, parents, top_links')
+ TopLink = namedtuple('TopLink', 'node_id, haplotype')
+ ParentLoci = namedtuple('ParentLoci', 'loci, haplotype, p')
+
+ header = [':START_ID(HAPLOTYPE)', ':END_ID(HAPLOTYPE)', 'CP:DOUBLE[]', ':TYPE']
+
+ FULL_LOCI, full_name_index_dict = loci_order(conf.get("loci_map"))
+
+
+ all_combo_list, nodes_plan_a, nodes_plan_b, top_nodes_plan_b = labels_for_grap(conf, FULL_LOCI, csvdir)
+ sequence = count(0)
+
+ loci_combo_map = {combo_list: defaultdict(lambda: HaplotypeNode(node_id=next(sequence),
+ freq_list=[], allele_list=[], parents=[],
+ top_links=set())) for combo_list in all_combo_list}
+
+ # for key in loci_combo_map:
+ # print (key)
+
+ # Load the frequency file into the FULL_LOCI dictionary with the cleaned up haplotype as the key.
+
+ haplist_overall = {} # list of haplotypes across all populations
+ pop_hap_combos = {}
+
+ num_of_alleles = len(FULL_LOCI)
+ with open(freq_file) as f:
+ for hap_line in f:
+ haplotype, pop, freq = hap_line.split(',')
+ if haplotype == "hap":
+ continue
+ freq = float(freq)
+ # Ignore lines with 0 freq
+ if freq == 0.0:
+ continue
+ # trimming frequencies
+ if freq < dict_count_of_pop[pop]:
+ continue
+
+ hap_list = make_allele_list(haplotype, full_name_index_dict, num_of_alleles)
+ haplotype = '~'.join(hap_list)
+ pop_haplotype = pop + '-' + haplotype
+ haplist_overall[haplotype] = 1
+ pop_hap_combos[pop_haplotype] = freq
+
+ for haplotype in haplist_overall:
+ # make frequency array
+ freqs = []
+ for pop in pops:
+ # check to see if population + haplotype combo exists
+ pop_haplotype = pop + '-' + haplotype
+ if pop_haplotype in pop_hap_combos:
+ freqs.append(pop_hap_combos[pop_haplotype])
+ else:
+ freqs.append(0)
+ hap_list = make_allele_list(haplotype, full_name_index_dict, num_of_alleles)
+ loci_combo_map[FULL_LOCI][haplotype] = HaplotypeNode(node_id=next(sequence),
+ freq_list=freqs,
+ allele_list=hap_list,
+ parents=None,
+ top_links=set())
+
+
+ # list(loci_combo_map[FULL_LOCI].items())[99]
+
+ # For each loci block, get all the allele combinations. Create parents link, and accumulate the freqs to create new Haplotype Nodes.
+
+ for loci in all_combo_list:
+ if loci != FULL_LOCI: # FULL_LOCI is already in the dictionary
+ loci_map = loci_combo_map[loci]
+ loci_indices = find_indices(FULL_LOCI, loci)
+ for full_hap_name, full_hap_node in loci_combo_map[FULL_LOCI].items():
+ allele_list = full_hap_node.allele_list
+ haplotype = '~'.join([allele_list[i] for i in loci_indices])
+ haplotype_node = loci_map[haplotype]
+
+ if loci in nodes_plan_b:# find parents just for internal nodes of plan B
+ parents = find_parents(FULL_LOCI, loci_indices, allele_list, full_hap_node.freq_list, ParentLoci, top_nodes_plan_b)
+ parents_list = haplotype_node.parents
+ for parent in parents:
+ parents_list.append(parent)
+ else:
+ parents_list = []
+
+ if loci in nodes_plan_a:# top link just for nodes of plan A
+ top_links = haplotype_node.top_links
+ top_links.add(TopLink(node_id=full_hap_node.node_id, haplotype=full_hap_name))
+ else:
+ top_links = None
+
+ # have to make freq_list with all zeros otherwise map add function returns empty list
+ new_freq_list = []
+ if not haplotype_node.freq_list:
+ for pop in pops:
+ new_freq_list.append(0)
+ else:
+ new_freq_list = haplotype_node.freq_list
+ freq_sum = list(map(add, new_freq_list, full_hap_node.freq_list))
+
+ new_node = HaplotypeNode(node_id=haplotype_node.node_id,
+ freq_list=freq_sum,
+ allele_list=None,
+ parents=parents_list,
+ top_links=top_links)
+
+ loci_map[haplotype] = new_node
+
+
+ # #### Build Nodes file
+
+ header = ['haplotypeId:ID(HAPLOTYPE)', 'name', 'loci:LABEL', 'frequency:DOUBLE[]']
+ node_file = csvdir + '/nodes.csv'
+ with open(node_file, mode='w') as csvfile:
+ csv_writer = csv.writer(csvfile)
+ csv_writer.writerow(header)
+ for loci in all_combo_list:
+ loci_map = loci_combo_map[loci]
+ for haplotype, haplotype_node in loci_map.items():
+ freq_array = ';'.join(map(str, haplotype_node.freq_list))
+ csv_writer.writerow([haplotype_node.node_id, haplotype, loci, freq_array])
+
+ # #### Build Edges File
+
+ edgeheader = [':START_ID(HAPLOTYPE)', ':END_ID(HAPLOTYPE)', 'CP:DOUBLE[]', ':TYPE']
+ edge_file = csvdir + '/edges.csv'
+ with open(edge_file, mode='w') as csvfile:
+ csv_writer = csv.writer(csvfile)
+ csv_writer.writerow(edgeheader)
+
+ for loci in nodes_plan_b:
+ if loci != FULL_LOCI: # FULL_LOCI is already in the dictionary
+ loci_map = loci_combo_map[loci]
+ for haplotype, haplotype_node in list(loci_map.items()):
+ for parent in haplotype_node.parents:
+ # assert(haplotype_node.freq > 0.0)
+ # avoid division by zero
+
+ cp = list(map(dividebyzero, parent.p, haplotype_node.freq_list))
+ prob_array = ";".join(map(str, cp))
+ loci_combo = parent.loci
+ hap = parent.haplotype
+ parent_id = loci_combo_map[loci_combo][hap].node_id
+ csv_writer.writerow([haplotype_node.node_id, parent_id, prob_array, 'CP'])
+
+ # #### Generate Top Links file
+
+ topheader = [':START_ID(HAPLOTYPE)', ':END_ID(HAPLOTYPE)', ':TYPE']
+ top_links_file = csvdir + '/top_links.csv'
+ with open(top_links_file, mode='w') as csvfile:
+ csv_writer = csv.writer(csvfile)
+ csv_writer.writerow(topheader)
+ for loci in nodes_plan_a:
+ if loci != FULL_LOCI: # FULL_LOCI is already in the dictionary
+ #
+ loci_map = loci_combo_map[loci]
+ for haplotype, haplotype_node in list(loci_map.items()):
+ top_links = haplotype_node.top_links
+ for top_link in top_links:
+ csv_writer.writerow([haplotype_node.node_id, top_link.node_id, 'TOP'])
+
+ # #### Generate Info Node file
+
+ infonode_header = ['INFO_NODE_ID:ID(INFO_NODE)', 'populations:STRING[]', 'INFO_NODE:LABEL']
+ top_links_file = csvdir + '/info_node.csv'
+ with open(top_links_file, mode='w') as csvfile:
+ csv_writer = csv.writer(csvfile)
+ csv_writer.writerow(infonode_header)
+ csv_writer.writerow([1, ";".join(pops), "INFO_NODE"])
+
+if __name__== "__main__":
+ generate_graph()
diff --git a/grim/imputation/graph_generation/generate_neo4j_single_hpf.py b/grim/imputation/graph_generation/generate_neo4j_single_hpf.py
new file mode 100755
index 0000000..d806bcc
--- /dev/null
+++ b/grim/imputation/graph_generation/generate_neo4j_single_hpf.py
@@ -0,0 +1,320 @@
+
+# coding: utf-8
+
+# #### TODO
+# * ~~Remove little g from allele names~~
+# * ~~Add toplinks~~
+# * ~~Set of ABCRQ in FULL_LOCI~~
+# * ~~Remove ChainMap~~
+# * ~~Use namedtuples~~
+# * ~~Ignore 0 HF frequency from Input file~~
+# * Compare with Matlab output
+# * Document the code
+# * Extract a module
+# * Accepts the list of FULL_LOCI and initial Freq. file
+
+from itertools import combinations, count
+from collections import defaultdict, namedtuple
+from operator import indexOf, add, truediv
+import csv
+import os
+import pandas as pd
+
+os.makedirs('output/csv', exist_ok=True)
+
+HaplotypeNode = namedtuple('HaplotypeNode', 'node_id, freq_list, allele_list, parents, top_links')
+TopLink = namedtuple('TopLink', 'node_id, haplotype')
+ParentLoci = namedtuple('ParentLoci', 'loci, haplotype, p')
+
+
+# #### nCr: n Choose r
+
+def nCr(nchars, r):
+ return [''.join(x) for x in combinations(nchars, r)]
+
+
+FULL_LOCI='ABCQR'
+
+
+all_combo_list = [FULL_LOCI]
+for i in range(len(FULL_LOCI) -1, 0, -1):
+ all_combo_list.extend(nCr(FULL_LOCI, i))
+len(all_combo_list)
+
+
+sequence = count(0)
+
+
+loci_combo_map = {combo_list: defaultdict(lambda : HaplotypeNode(node_id = next(sequence),
+ freq_list = [],
+ allele_list=[],
+ parents=[],
+ top_links=set()))
+ for combo_list in all_combo_list }
+
+
+# #### Verify
+# There should be 31 loci combos. The keys should be all the different combinations.
+
+print(len(loci_combo_map))
+loci_combo_map.keys()
+
+
+def make_allele_list(haplotype):
+ hl = haplotype.split('~')
+ # Remove g from the allele names
+ hl = list(map(lambda h: h[:-1] if(h[-1] == 'g') else h, hl))
+ hl.sort()
+ return hl
+
+
+# #### Verify
+# Test that the make_allele_list works
+
+
+# #### Load Frequency file
+# Load the frequency file into the FULL_LOCI dictionary with the cleaned up haplotype as the key.
+
+haplist_overall = {} # list of haplotypes across all populations
+pop_hap_combos = {}
+pops = {}
+# pops = ['AAFA','AFB','AINDI','AISC','ALANAM','AMIND','CARB','CARHIS','CARIBI','FILII','KORI','JAPI','MENAFC',
+# 'NAMER','NCHI','SCAHIS','SCAMB','SCSEAI','VIET','AFA','API','CAU','HIS','NAM']
+# pops = ['VIET', 'KORI', 'JAPI', 'FILII']
+# pops = ['VIET', 'FILII']
+freqfile = 'output/hpf.csv'
+with open(freqfile) as f:
+ for hap_line in f:
+ haplotype, pop, freq = hap_line.split(',')
+ if haplotype == "hap":
+ continue
+ freq = float(freq)
+ # Ignore lines with 0 freq
+ if freq == 0.0:
+ continue
+ #if freq <