Skip to content

Commit

Permalink
Fixed BiGG conversion scripts.
Browse files Browse the repository at this point in the history
Moved SQLite DB out of the project to a Dropbox folder, configureSQLiteDB fetches it automatically.
Moved dev folder out of resources.
Updated BiGG to version 1.5.
  • Loading branch information
mephenor committed Jun 21, 2018
1 parent 6d63b18 commit 742383e
Show file tree
Hide file tree
Showing 12 changed files with 131 additions and 60 deletions.
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -5,6 +5,7 @@
.project
/build/
/target/
/resources/edu/ucsd/sbrg/bigg/bigg.zip
/resources/edu/ucsd/sbrg/bigg/bigg.sqlite
/scripts/ModelPolisherTemplate.*
ModelPolisher.iml
Expand Down
14 changes: 11 additions & 3 deletions build.gradle
Expand Up @@ -85,6 +85,13 @@ task bareJar(type: Jar) {
with jar
}

// without dependencies, bigg.sqlite included
task slimJar(type: Jar) {
baseName = project.name + "-slim"
with jar
}


// zip lib folder for release
task zipLibs(type: Zip) {
from "lib"
Expand All @@ -111,9 +118,10 @@ task release() {
dependsOn tasks["zipLibs"]
dependsOn tasks["zipScripts"]
// necessary, as order is not defined by dependsOn
fatJar.mustRunAfter classes
bareJar.mustRunAfter fatJar
lightJar.mustRunAfter bareJar
bareJar.mustRunAfter classes
// slimJar.mustRunAfter bareJar
lightJar.mustRunAfter slimJar
fatJar.mustRunAfter lightJar
}

// clean up target directory
Expand Down
File renamed without changes.
File renamed without changes.
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
3 changes: 1 addition & 2 deletions gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,5 @@
#Fri Jun 02 13:13:23 CEST 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip
6 changes: 3 additions & 3 deletions gradlew
Expand Up @@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"

warn ( ) {
warn () {
echo "$*"
}

die ( ) {
die () {
echo
echo "$*"
echo
Expand Down Expand Up @@ -155,7 +155,7 @@ if $cygwin ; then
fi

# Escape application args
save ( ) {
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
Expand Down
Binary file removed resources/edu/ucsd/sbrg/bigg/bigg.zip
Binary file not shown.
139 changes: 88 additions & 51 deletions scripts/cleanup_dump.py
@@ -1,77 +1,114 @@
#!/bin/python3

import re
import os

def main():
parens = re.compile("\(.*\);")
using = re.compile("USING.*")

with open("bigg.sql") as file:
content = file.readlines()
output = []
for line in content:
if line.startswith("--") or line.startswith("SET") or line.startswith("COPY") or "SELECT pg_catalog.setval(" in line:
line = ""
line = line.replace("true", "1")
line = line.replace("false", "0")
output.append(line)
with open("converted.sql", "w") as db:
db.write("".join(output))

parens = re.compile("\(.*\);")
using = re.compile("USING.*")
def main():
clean_schema()
clean_data()


def clean_schema():
with open("bigg_schema.sql") as file:
content = file.readlines()
output = []
type_block = False
block = False
function_block = False
function_end = False
sequence_block = False
alter_table_block = False
found_cobra = False
found_start = False
for line in content:
if type_block:
if ");" in line:
type_block = False
elif function_block:

# only use the public schema for now
if not found_cobra:
if line.startswith("SET search_path = cobradb, pg_catalog;"):
found_cobra = True
continue
if not found_start:
if line.startswith("SET search_path = public, pg_catalog;"):
found_start = True
continue

if function_block:
if function_end and line.strip() == '':
function_block = False
function_end = False
if not line.startswith(" ") and ";" in line:
function_end = True
elif sequence_block:
if ";" in line:
sequence_block = False
elif alter_table_block:
continue

elif block:
if ";" in line:
alter_table_block = False
elif line.startswith("--") or line.startswith("SET") or line.startswith("COPY") or "SELECT pg_catalog.setval(" in line\
or line.startswith("CREATE EXTENSION") or line.startswith("COMMENT") or "OWNER" in line:
pass
elif line.startswith("CREATE TYPE") or line.startswith("ALTER TYPE"):
if not line.endswith(";"):
type_block = True
elif line.startswith("CREATE FUNCTION") or line.startswith("ALTER FUNCTION"):
if not line.endswith(";"):
function_block = True
elif line.startswith("CREATE SEQUENCE") or line.startswith("ALTER SEQUENCE"):
if not line.endswith(";"):
sequence_block = True
elif line.startswith("ALTER TABLE ONLY"):
if not line.endswith(";"):
alter_table_block = True
block = False
continue

elif is_var(line, "TYPE") or is_var(line, "SEQUENCE") or line.startswith("ALTER TABLE ONLY"):
block = is_block(line)
continue

elif is_var(line, "FUNCTION"):
function_block = is_block(line)
continue

elif line.startswith("CREATE INDEX"):
line = line.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS")
matches = re.search(parens, line)
match = matches.group(0)
match = match.replace(" gin_trgm_ops", "")
line = re.sub(using, match, line)
line = fix_index(line)
elif line.startswith("--") \
or line.strip() == "" \
or line.startswith("SET") \
or line.startswith("COPY") \
or line.startswith("CREATE EXTENSION") \
or line.startswith("COMMENT") \
or "OWNER" in line \
or "SELECT pg_catalog.setval(" in line \
or "SCHEMA" in line:
line = ""

line = line.replace("true", "1")
line = line.replace("false", "0")
if line.strip() != "":
output.append(line)
else:
line = line.replace("true", "1")
line = line.replace("false", "0")
if line.strip() != "":
output.append(line)

with open("schema.sql", "w") as db:
db.write("".join(output))


def is_var(line, var):
return line.startswith("CREATE {}".format(var)) or line.startswith("ALTER {}".format(var))


def is_block(line):
return not line.endswith(";")


def fix_index(line):
line = line.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS")
matches = re.search(parens, line)
match = matches.group(0)
match = match.replace(" gin_trgm_ops", "")
return re.sub(using, match, line)


def clean_data():
with open("bigg.sql") as file:
content = file.readlines()
output = []
for line in content:
if line.startswith("--") \
or line.startswith("SET") \
or line.startswith("COPY") \
or "SELECT pg_catalog.setval(" in line \
or "database_version" in line:
line = ""
line = line.replace("true", "1")
line = line.replace("false", "0")
if line.strip() != "":
output.append(line)
with open("converted.sql", "w") as db:
db.write("".join(output))


main()
6 changes: 6 additions & 0 deletions scripts/configureSQLiteDB.sh
@@ -1,3 +1,9 @@
#!/bin/bash

if [ ! -f ../resources/edu/ucsd/sbrg/bigg/bigg.zip ]; then
curl -L -o bigg.zip https://www.dropbox.com/s/0j3pozc7a0p2b94/bigg.zip\?dl\=0
mv bigg.zip ../resources/edu/ucsd/sbrg/bigg
fi

unzip -od ../resources/edu/ucsd/sbrg/bigg ../resources/edu/ucsd/sbrg/bigg/bigg.zip
python3 createIndices.py
2 changes: 1 addition & 1 deletion scripts/convertToSqlite.sh
Expand Up @@ -12,4 +12,4 @@ echo "Started conversion of local PostgreSQL BiGGDB to SQLite..." \
&& mv bigg.sqlite ../resources/edu/ucsd/sbrg/bigg \
&& echo "Removing temporary files..." \
&& rm bigg.sql converted.sql bigg_schema.sql schema.sql \
&& echo "Finished."
&& printf "Finished.\nYou can now create the indices using createIndices.py."
20 changes: 20 additions & 0 deletions scripts/createIndices.py
@@ -0,0 +1,20 @@
#!/bin/python3

import sqlite3


def main():
connection = sqlite3.connect("../resources/edu/ucsd/sbrg/bigg/bigg.sqlite")
cursor = connection.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = cursor.fetchall()
for table in tables:
table = table[0]
cursor.execute("SELECT * FROM " + table)
for (pos, column) in enumerate(cursor.description):
column = column[0]
query = "CREATE INDEX IF NOT EXISTS " + table + "_idx" + str(pos) + " on " + table + "(" + column + ")"
cursor.execute(query)


main()

0 comments on commit 742383e

Please sign in to comment.