Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Moved SQLite DB out of the project to a Dropbox folder, configureSQLiteDB fetches it automatically. Moved dev folder out of resources. Updated BiGG to version 1.5.
- Loading branch information
Showing
12 changed files
with
131 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,5 @@ | ||
#Fri Jun 02 13:13:23 CEST 2017 | ||
distributionBase=GRADLE_USER_HOME | ||
distributionPath=wrapper/dists | ||
distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip | ||
zipStoreBase=GRADLE_USER_HOME | ||
zipStorePath=wrapper/dists | ||
distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,77 +1,114 @@ | ||
#!/bin/python3 | ||
|
||
import re | ||
import os | ||
|
||
def main(): | ||
parens = re.compile("\(.*\);") | ||
using = re.compile("USING.*") | ||
|
||
with open("bigg.sql") as file: | ||
content = file.readlines() | ||
output = [] | ||
for line in content: | ||
if line.startswith("--") or line.startswith("SET") or line.startswith("COPY") or "SELECT pg_catalog.setval(" in line: | ||
line = "" | ||
line = line.replace("true", "1") | ||
line = line.replace("false", "0") | ||
output.append(line) | ||
with open("converted.sql", "w") as db: | ||
db.write("".join(output)) | ||
|
||
parens = re.compile("\(.*\);") | ||
using = re.compile("USING.*") | ||
def main(): | ||
clean_schema() | ||
clean_data() | ||
|
||
|
||
def clean_schema(): | ||
with open("bigg_schema.sql") as file: | ||
content = file.readlines() | ||
output = [] | ||
type_block = False | ||
block = False | ||
function_block = False | ||
function_end = False | ||
sequence_block = False | ||
alter_table_block = False | ||
found_cobra = False | ||
found_start = False | ||
for line in content: | ||
if type_block: | ||
if ");" in line: | ||
type_block = False | ||
elif function_block: | ||
|
||
# only use the public schema for now | ||
if not found_cobra: | ||
if line.startswith("SET search_path = cobradb, pg_catalog;"): | ||
found_cobra = True | ||
continue | ||
if not found_start: | ||
if line.startswith("SET search_path = public, pg_catalog;"): | ||
found_start = True | ||
continue | ||
|
||
if function_block: | ||
if function_end and line.strip() == '': | ||
function_block = False | ||
function_end = False | ||
if not line.startswith(" ") and ";" in line: | ||
function_end = True | ||
elif sequence_block: | ||
if ";" in line: | ||
sequence_block = False | ||
elif alter_table_block: | ||
continue | ||
|
||
elif block: | ||
if ";" in line: | ||
alter_table_block = False | ||
elif line.startswith("--") or line.startswith("SET") or line.startswith("COPY") or "SELECT pg_catalog.setval(" in line\ | ||
or line.startswith("CREATE EXTENSION") or line.startswith("COMMENT") or "OWNER" in line: | ||
pass | ||
elif line.startswith("CREATE TYPE") or line.startswith("ALTER TYPE"): | ||
if not line.endswith(";"): | ||
type_block = True | ||
elif line.startswith("CREATE FUNCTION") or line.startswith("ALTER FUNCTION"): | ||
if not line.endswith(";"): | ||
function_block = True | ||
elif line.startswith("CREATE SEQUENCE") or line.startswith("ALTER SEQUENCE"): | ||
if not line.endswith(";"): | ||
sequence_block = True | ||
elif line.startswith("ALTER TABLE ONLY"): | ||
if not line.endswith(";"): | ||
alter_table_block = True | ||
block = False | ||
continue | ||
|
||
elif is_var(line, "TYPE") or is_var(line, "SEQUENCE") or line.startswith("ALTER TABLE ONLY"): | ||
block = is_block(line) | ||
continue | ||
|
||
elif is_var(line, "FUNCTION"): | ||
function_block = is_block(line) | ||
continue | ||
|
||
elif line.startswith("CREATE INDEX"): | ||
line = line.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS") | ||
matches = re.search(parens, line) | ||
match = matches.group(0) | ||
match = match.replace(" gin_trgm_ops", "") | ||
line = re.sub(using, match, line) | ||
line = fix_index(line) | ||
elif line.startswith("--") \ | ||
or line.strip() == "" \ | ||
or line.startswith("SET") \ | ||
or line.startswith("COPY") \ | ||
or line.startswith("CREATE EXTENSION") \ | ||
or line.startswith("COMMENT") \ | ||
or "OWNER" in line \ | ||
or "SELECT pg_catalog.setval(" in line \ | ||
or "SCHEMA" in line: | ||
line = "" | ||
|
||
line = line.replace("true", "1") | ||
line = line.replace("false", "0") | ||
if line.strip() != "": | ||
output.append(line) | ||
else: | ||
line = line.replace("true", "1") | ||
line = line.replace("false", "0") | ||
if line.strip() != "": | ||
output.append(line) | ||
|
||
with open("schema.sql", "w") as db: | ||
db.write("".join(output)) | ||
|
||
|
||
def is_var(line, var): | ||
return line.startswith("CREATE {}".format(var)) or line.startswith("ALTER {}".format(var)) | ||
|
||
|
||
def is_block(line): | ||
return not line.endswith(";") | ||
|
||
|
||
def fix_index(line): | ||
line = line.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS") | ||
matches = re.search(parens, line) | ||
match = matches.group(0) | ||
match = match.replace(" gin_trgm_ops", "") | ||
return re.sub(using, match, line) | ||
|
||
|
||
def clean_data(): | ||
with open("bigg.sql") as file: | ||
content = file.readlines() | ||
output = [] | ||
for line in content: | ||
if line.startswith("--") \ | ||
or line.startswith("SET") \ | ||
or line.startswith("COPY") \ | ||
or "SELECT pg_catalog.setval(" in line \ | ||
or "database_version" in line: | ||
line = "" | ||
line = line.replace("true", "1") | ||
line = line.replace("false", "0") | ||
if line.strip() != "": | ||
output.append(line) | ||
with open("converted.sql", "w") as db: | ||
db.write("".join(output)) | ||
|
||
|
||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,9 @@ | ||
#!/bin/bash | ||
|
||
if [ ! -f ../resources/edu/ucsd/sbrg/bigg/bigg.zip ]; then | ||
curl -L -o bigg.zip https://www.dropbox.com/s/0j3pozc7a0p2b94/bigg.zip\?dl\=0 | ||
mv bigg.zip ../resources/edu/ucsd/sbrg/bigg | ||
fi | ||
|
||
unzip -od ../resources/edu/ucsd/sbrg/bigg ../resources/edu/ucsd/sbrg/bigg/bigg.zip | ||
python3 createIndices.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
#!/bin/python3 | ||
|
||
import sqlite3 | ||
|
||
|
||
def main(): | ||
connection = sqlite3.connect("../resources/edu/ucsd/sbrg/bigg/bigg.sqlite") | ||
cursor = connection.cursor() | ||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") | ||
tables = cursor.fetchall() | ||
for table in tables: | ||
table = table[0] | ||
cursor.execute("SELECT * FROM " + table) | ||
for (pos, column) in enumerate(cursor.description): | ||
column = column[0] | ||
query = "CREATE INDEX IF NOT EXISTS " + table + "_idx" + str(pos) + " on " + table + "(" + column + ")" | ||
cursor.execute(query) | ||
|
||
|
||
main() |