diff --git a/common-script b/common-script index 9c05f8ed3..b3e987abd 100755 --- a/common-script +++ b/common-script @@ -72,6 +72,8 @@ do_completeness_sqlite() { if [[ "${HAS_GROUP_PARAM}" == "1" ]]; then bash scripts/sqlite/completeness-grouped.sqlite.sh ${OUTPUT_DIR} + else + bash scripts/sqlite/completeness.sqlite.sh ${OUTPUT_DIR} fi } diff --git a/scripts/sqlite/completeness-grouped.sqlite.sh b/scripts/sqlite/completeness-grouped.sqlite.sh index a23ea64f9..598d9d29b 100755 --- a/scripts/sqlite/completeness-grouped.sqlite.sh +++ b/scripts/sqlite/completeness-grouped.sqlite.sh @@ -13,9 +13,9 @@ log() { OUTPUT_DIR=$1 -log "create table grouped_marc_elements" +log "create table marc_elements_grouped" sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF -CREATE TABLE IF NOT EXISTS "grouped_marc_elements" ( +CREATE TABLE IF NOT EXISTS "marc_elements_grouped" ( "groupId" INTEGER, "documenttype" TEXT, "path" TEXT, @@ -31,13 +31,13 @@ CREATE TABLE IF NOT EXISTS "grouped_marc_elements" ( "stddev" REAL, "histogram" TEXT ); -CREATE INDEX IF NOT EXISTS "gme_groupId" ON "grouped_marc_elements" ("groupId"); -CREATE INDEX IF NOT EXISTS "gme_documenttype" ON "grouped_marc_elements" ("documenttype"); +CREATE INDEX IF NOT EXISTS "meg_groupId" ON "marc_elements_grouped" ("groupId"); +CREATE INDEX IF NOT EXISTS "meg_documenttype" ON "marc_elements_grouped" ("documenttype"); EOF -log "clean grouped_marc_elements" +log "clean marc_elements_grouped" sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF -DELETE FROM grouped_marc_elements; +DELETE FROM marc_elements_grouped; EOF log "create headless CSV" @@ -46,7 +46,7 @@ tail -n +2 ${OUTPUT_DIR}/completeness-grouped-marc-elements.csv > ${OUTPUT_DIR}/ log "import marc elements" sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF .mode csv -.import ${OUTPUT_DIR}/completeness-grouped-marc-elements-noheader.csv grouped_marc_elements +.import ${OUTPUT_DIR}/completeness-grouped-marc-elements-noheader.csv marc_elements_grouped EOF log "drop headless CSV" diff --git a/scripts/sqlite/completeness.sqlite.sh b/scripts/sqlite/completeness.sqlite.sh new file mode 100755 index 000000000..3389f2fed --- /dev/null +++ b/scripts/sqlite/completeness.sqlite.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Strores marc-elements.csv into SQLite +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# + +log() { + timestamp=`date +"%F %T"` + echo -en "\033[0D\033[1;37m$timestamp>\033[0m " + echo "$1" +} + +OUTPUT_DIR=$1 + +log "create table marc_elements" +sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF +CREATE TABLE IF NOT EXISTS "marc_elements" ( + "documenttype" TEXT, + "path" TEXT, + "packageid" INTEGER, + "package" TEXT, + "tag" TEXT, + "subfield" TEXT, + "number-of-record" INTEGER, + "number-of-instances" INTEGER, + "min" INTEGER, + "max" INTEGER, + "mean" REAL, + "stddev" REAL, + "histogram" TEXT +); +CREATE INDEX IF NOT EXISTS "gme_groupId" ON "marc_elements" ("groupId"); +CREATE INDEX IF NOT EXISTS "gme_documenttype" ON "marc_elements" ("documenttype"); +EOF + +log "clean marc_elements" +sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF +DELETE FROM marc_elements; +EOF + +log "create headless CSV" +tail -n +2 ${OUTPUT_DIR}/marc-elements.csv > ${OUTPUT_DIR}/marc-elements-noheader.csv + +log "import marc elements" +sqlite3 ${OUTPUT_DIR}/qa_catalogue.sqlite << EOF +.mode csv +.import ${OUTPUT_DIR}/marc-elements-noheader.csv marc_elements +EOF + +log "drop headless CSV" +rm ${OUTPUT_DIR}/marc-elements-noheader.csv \ No newline at end of file