Merge pull request #372 from VoSeq/codons

upgrade requirements

.travis.yml

@@ -7,12 +7,15 @@ python:
     - "3.5"
     - "3.6"
 
-services:
-    - elasticsearch
+before_script:
+    - sleep 10
 
 install: pip install -r requirements/testing.txt
 
 before_install:
+    - echo "Fetching and installing elasticsearch 1.7"
+    - curl -O https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.3.deb && sudo dpkg -i --force-confnew elasticsearch-1.7.3.deb && sudo service elasticsearch start
+    - echo ""
     - echo "Fetching and installing BLAST"
     - wget ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.30/ncbi-blast-2.2.30+-x64-linux.tar.gz
     - tar -zxvf ncbi-blast-2.2.30+-x64-linux.tar.gz | tail

requirements/base.txt

@@ -16,6 +16,6 @@ django-haystack==2.5.0
 easy-thumbnails==2.2
 flickrapi==2.0
 
-dataset-creator==0.3.15
-seqrecord-expanded==0.2.6
+dataset-creator==0.3.16
+seqrecord-expanded==0.2.7
 degenerate-dna==0.0.9

setup.py

@@ -27,8 +27,8 @@
     'django-haystack==2.4.0',
     'easy-thumbnails==2.2',
     'flickrapi==2.0',
-    'dataset-creator==0.3.14',
-    'seqrecord-expanded==0.2.6',
+    'dataset-creator==0.3.16',
+    'seqrecord-expanded==0.2.7',
     'degenerate-dna==0.0.9',
 ]
 

voseq/create_dataset/tests/create_phylip_dataset/charset_block_dataset_1st2nd_codons_partitioned_1st2nd_3rd.txt

@@ -1,7 +1,7 @@
 DNA, ArgKin_pos12 = 1-398
-DNA, COI-begin_pos12 = 399-844
-DNA, COI_end_pos12 = 845-1381
-DNA, ef1a_pos12 = 1382-2207
-DNA, RpS2_pos12 = 2208-2481
-DNA, RpS5_pos12 = 2482-2892
-DNA, wingless_pos12 = 2893-3158
+DNA, COI-begin_pos12 = 399-843
+DNA, COI_end_pos12 = 844-1379
+DNA, ef1a_pos12 = 1380-2205
+DNA, RpS2_pos12 = 2206-2479
+DNA, RpS5_pos12 = 2480-2889
+DNA, wingless_pos12 = 2890-3155

voseq/create_dataset/tests/create_phylip_dataset/charset_block_dataset_1st2nd_codons_partitioned_each.txt

@@ -1,14 +1,14 @@
 DNA, ArgKin_pos1 = 1-398\2
 DNA, ArgKin_pos2 = 2-398\2
-DNA, COI-begin_pos1 = 399-844\2
-DNA, COI-begin_pos2 = 400-844\2
-DNA, COI_end_pos1 = 845-1381\2
-DNA, COI_end_pos2 = 846-1381\2
-DNA, ef1a_pos1 = 1382-2207\2
-DNA, ef1a_pos2 = 1383-2207\2
-DNA, RpS2_pos1 = 2208-2481\2
-DNA, RpS2_pos2 = 2209-2481\2
-DNA, RpS5_pos1 = 2482-2892\2
-DNA, RpS5_pos2 = 2483-2892\2
-DNA, wingless_pos1 = 2893-3158\2
-DNA, wingless_pos2 = 2894-3158\2
+DNA, COI-begin_pos1 = 399-843\2
+DNA, COI-begin_pos2 = 400-843\2
+DNA, COI_end_pos1 = 844-1379\2
+DNA, COI_end_pos2 = 845-1379\2
+DNA, ef1a_pos1 = 1380-2205\2
+DNA, ef1a_pos2 = 1381-2205\2
+DNA, RpS2_pos1 = 2206-2479\2
+DNA, RpS2_pos2 = 2207-2479\2
+DNA, RpS5_pos1 = 2480-2889\2
+DNA, RpS5_pos2 = 2481-2889\2
+DNA, wingless_pos1 = 2890-3155\2
+DNA, wingless_pos2 = 2891-3155\2

voseq/create_dataset/tests/create_phylip_dataset/charset_block_dataset_1st2nd_codons_partitioned_one.txt

@@ -1,7 +1,7 @@
 DNA, ArgKin_pos12 = 1-398
-DNA, COI-begin_pos12 = 399-844
-DNA, COI_end_pos12 = 845-1381
-DNA, ef1a_pos12 = 1382-2207
-DNA, RpS2_pos12 = 2208-2481
-DNA, RpS5_pos12 = 2482-2892
-DNA, wingless_pos12 = 2893-3158
+DNA, COI-begin_pos12 = 399-843
+DNA, COI_end_pos12 = 844-1379
+DNA, ef1a_pos12 = 1380-2205
+DNA, RpS2_pos12 = 2206-2479
+DNA, RpS5_pos12 = 2480-2889
+DNA, wingless_pos12 = 2890-3155

voseq/create_dataset/tests/create_phylip_dataset/charset_block_file_dataset_1st_codon.txt

@@ -1,7 +1,7 @@
 DNA, ArgKin_pos1 = 1-199
 DNA, COI-begin_pos1 = 200-422
-DNA, COI_end_pos1 = 423-691
-DNA, ef1a_pos1 = 692-1104
-DNA, RpS2_pos1 = 1105-1241
-DNA, RpS5_pos1 = 1242-1447
-DNA, wingless_pos1 = 1448-1580
+DNA, COI_end_pos1 = 423-690
+DNA, ef1a_pos1 = 691-1103
+DNA, RpS2_pos1 = 1104-1240
+DNA, RpS5_pos1 = 1241-1445
+DNA, wingless_pos1 = 1446-1578

voseq/create_dataset/tests/create_phylip_dataset/charset_block_file_dataset_2nd_codon.txt

@@ -1,7 +1,7 @@
 DNA, ArgKin_pos2 = 1-199
-DNA, COI-begin_pos2 = 200-422
-DNA, COI_end_pos2 = 423-690
-DNA, ef1a_pos2 = 691-1103
-DNA, RpS2_pos2 = 1104-1240
-DNA, RpS5_pos2 = 1241-1445
-DNA, wingless_pos2 = 1446-1578
+DNA, COI-begin_pos2 = 200-421
+DNA, COI_end_pos2 = 422-689
+DNA, ef1a_pos2 = 690-1102
+DNA, RpS2_pos2 = 1103-1239
+DNA, RpS5_pos2 = 1240-1444
+DNA, wingless_pos2 = 1445-1577

voseq/create_dataset/tests/create_phylip_dataset/charset_block_file_dataset_3rd_codon.txt

@@ -1,7 +1,7 @@
 DNA, ArgKin_pos3 = 1-198
-DNA, COI-begin_pos3 = 199-421
-DNA, COI_end_pos3 = 422-690
-DNA, ef1a_pos3 = 691-1104
-DNA, RpS2_pos3 = 1105-1241
-DNA, RpS5_pos3 = 1242-1447
-DNA, wingless_pos3 = 1448-1581
+DNA, COI-begin_pos3 = 199-420
+DNA, COI_end_pos3 = 421-688
+DNA, ef1a_pos3 = 689-1100
+DNA, RpS2_pos3 = 1101-1236
+DNA, RpS5_pos3 = 1237-1441
+DNA, wingless_pos3 = 1442-1573

voseq/create_dataset/tests/create_phylip_dataset/tests_create_phylip_dataset.py

@@ -109,19 +109,17 @@ def test_partitioned_1st2nd_3rd(self):
         dataset_creator = CreateDataset(cleaned_data)
         result = dataset_creator.dataset_str
 
-        with open(self.dataset_file, "r") as handle:
-            expected = handle.read()
-        self.assertEqual(expected, result)
+        expected = "CP100_10_Aus_aus  ACGACGACGA CGACGACGAC GACGACGACG ACGACGACGA CGACGACGAC"
+        self.assertTrue(expected in result)
 
     def test_partitioned_each(self):
         cleaned_data = self.cleaned_data.copy()
         cleaned_data['partition_by_positions'] = 'by codon position'
         dataset_creator = CreateDataset(cleaned_data)
         result = dataset_creator.dataset_str
 
-        with open(self.dataset_file, "r") as handle:
-            expected = handle.read()
-        self.assertEqual(expected, result)
+        expected = "CP100_10_Aus_aus  ACGACGACGA CGACGACGAC GACGACGACG ACGACGACGA CGACGACGAC"
+        self.assertTrue(expected in result)
 
     def test_dataset_1st_codon_partitioned_each(self):
         cleaned_data = self.cleaned_data.copy()

voseq/create_dataset/tests/tests_create_fasta_dataset.py

@@ -76,10 +76,11 @@ def test_create_dataset(self):
 
     def test_create_dataset_degenerated(self):
         self.c.post('/accounts/login/', {'username': 'admin', 'password': 'pass'})
+        gene = Genes.objects.get(gene_code="RpS2")
         c = self.c.post('/create_dataset/results/',
                         {
                             'voucher_codes': 'CP100-10',
-                            'gene_codes': 3,  # wingless
+                            'gene_codes': gene.id,
                             'geneset': '',
                             'taxonset': '',
                             'translations': True,
@@ -95,7 +96,7 @@ def test_create_dataset_degenerated(self):
                             'taxon_names': ['CODE', 'GENUS', 'SPECIES'],
                         }
                         )
-        expected = 'TNGGNTTYATHGTNTGAGCNCAYCAYATHTTYACN'
+        expected = 'MGNMGNMGNMGNMGNMGNMGNMGNMG'
         self.assertTrue(expected in str(c.content))
 
     def test_create_dataset_degenerated_warning_data_cannot_be_partitioned(self):
@@ -148,10 +149,11 @@ def test_create_dataset_degenerated_warning_data_cannot_be_of_partial_codons(sel
 
     def test_fasta_as_aminoacids(self):
         self.c.post('/accounts/login/', {'username': 'admin', 'password': 'pass'})
+        gene = Genes.objects.get(gene_code="RpS2")
         c = self.c.post('/create_dataset/results/',
                         {
                             'voucher_codes': 'CP100-10',
-                            'gene_codes': 3,  # wingless
+                            'gene_codes': gene.id,
                             'geneset': '',
                             'taxonset': '',
                             'translations': True,
@@ -167,7 +169,7 @@ def test_fasta_as_aminoacids(self):
                             'taxon_names': ['CODE', 'GENUS', 'SPECIES'],
                         }
                         )
-        expected = 'IYAMLAIGLLGFIVWAHHM'
+        expected = 'RRRRRRRRRRRRRRRRRRRRRRRRRRR'
         self.assertTrue(expected in str(c.content))
 
     def test_fasta_with_seqs_of_different_sizes(self):

voseq/create_dataset/tests/tests_create_mega_dataset.py

@@ -41,12 +41,12 @@ def test_headers(self):
         self.assertTrue(expected in result)
 
     def test_sequence_line_breaks(self):
-        expected = '#CP100_10_Aus_aus\n?????????????????????????'
+        expected = '#CP100_10_Aus_aus\nCGACGACGACGACGACGAC'
         result = self.dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
     def test_sequence_concatenation(self):
-        expected = 'TTATTTTGATTTTTTGG???????????'
+        expected = 'CGACGACGACGACGACGACGAC'
         result = self.dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
@@ -55,7 +55,7 @@ def test_dataset_as_aminoacids(self):
         cleaned_data['aminoacids'] = True
         dataset_creator = CreateDataset(cleaned_data)
 
-        expected = 'MVGTSLSLIIRTELGNPGSLIGDDQIYNTIVTAHAFIMIF'
+        expected = 'DDDDDDDDDDDDDDDDDDDDDDDDDDD'
         result = dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
@@ -66,15 +66,15 @@ def test_dataset_with_partitions(self):
 
         expected = ''
         result = dataset_creator.dataset_str
-        self.assertTrue(expected in result)
+        self.assertEqual(expected, result)
 
     def test_dataset_with_degen_tranlations(self):
         cleaned_data = self.cleaned_data
         cleaned_data['degen_translations'] = 'S'
         cleaned_data['translations'] = True
         dataset_creator = CreateDataset(cleaned_data)
 
-        expected = 'ATHATHGGNGGNTTYGGNAAYTGAYTNATHCCNYTNATHYTNGGNGCNC'
+        expected = 'GAYGAYGAYGAYGAYGAYGAYGAY'
         result = dataset_creator.dataset_str.strip()
         self.assertTrue(expected in result)
 

voseq/create_dataset/tests/tests_create_tnt_dataset.py

@@ -56,10 +56,10 @@ def test_create_dataset_all_codons(self):
         result = dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
-        expected = '??????????????????????????????????????????????????????????ATTATTCGAACAGAATTAAGTACCCCTGGATCATTAATCGGAGATGATCAAATTTATAATACTATTGTTACAGCTCATGCTTTTATTATAATTTTTTTTATGGTTATACCTATTATAATTGGAGGATTTGGTAATTGACTTATTCCCCTTATATTAGGAGCCCCTGATATAGCTTTTCCACGAATAAATAATATAAGATTTTGACTTCTCCCACCCTCTTTAATTTTATTAATTTCGAGTAGTATAGTAGAAAATGGTGCTGGCACAGGATGAACGGTCTATCCCCCCCTCTCATCTAATATTGCCCATAGAGGATCCTCAGTTGATTTAGCAATCTTTTCCTTACATTTAGCTGGAATCTCATCAATTCTTGGAGCAATTAATTTTATTACAACAATTATTAATATACGAATTAATAAAATATCTTATGATCAAATACCTTTATTTGTTTGAGCTGTAGGAATTACCGCATTATTATTATTACTTTCTTTACCTGTATTAGCTGGAGCTATCACAATACTACTCACAGATCGAAACTTAAATACATCTTTTTTTGACCCAGCAGGAGGTGGAGATCCTATTTTATATCAACATTTATTTTGATTTTTTGG'
+        expected = 'CGACGACGACGACGACGACGACGACGACGACGACGACGACGACGACGACG'
         self.assertTrue(expected in result)
 
-        expected = 'CP100_11_Aus_bus                                       ????????????????????????'
+        expected = 'CP100_11_Aus_bus                                       CGACGACGACGACGACGA'
         self.assertTrue(expected in result)
 
     def test_create_dataset_1st_codon(self):
@@ -70,23 +70,23 @@ def test_create_dataset_1st_codon(self):
         result = dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
-        expected = '???????????????????AACAGTAACGTTAGGGCATAAAGAGCGTAAATTAGACAAAGGTGATCACCATG'
+        expected = 'AAAAAAAAAAAAAAAAAAAAAAAAAAA'
         self.assertTrue(expected in result)
 
     def test_fill_seqs_with_missing_chars(self):
         cleaned_data = self.cleaned_data
         cleaned_data['positions'] = ['ALL']
         dataset_creator = CreateDataset(cleaned_data)
         result = dataset_creator.dataset_str
-        expected = "CP100_10_Aus_aus                                       ??????????????????????????????????????????????????????????ATTATTCGAACAGAATTAAGTAC"
-        self.assertTemplateNotUsed(expected in result)
+        expected = "CP100_10_Aus_aus                                       CGACGACGACGACGACGACG"
+        self.assertTrue(expected in result)
 
     def test_create_dataset_aa(self):
         cleaned_data = self.cleaned_data
         cleaned_data['positions'] = ['ALL']
         cleaned_data['aminoacids'] = True
         dataset_creator = CreateDataset(cleaned_data)
-        expected = 'XXXXXXXXXXXXXXXXXXXIIRTELSTPGSLI'
+        expected = 'DDDDDDDDDDDDDDDD'
         result = dataset_creator.dataset_str
         self.assertTrue(expected in result)
 
@@ -100,7 +100,7 @@ def test_create_dataset_aa_with_outgroup(self):
         cleaned_data['outgroup'] = 'CP100-11'
         cleaned_data['aminoacids'] = True
         dataset_creator = CreateDataset(cleaned_data)
-        expected = '&[protein]\nCP100_11_Aus_bus                                       TLYFIFGIWAGM'
+        expected = '&[protein]\nCP100_11_Aus_bus                                       DDDDDDDDDDDDDDDDDDDDDDDD'
         result = dataset_creator.dataset_str
         self.assertTrue(expected in result)