Skip to content

Commit

Permalink
sql join bug fix; add "make check"; clean up
Browse files Browse the repository at this point in the history 
add "make check" synonym for "make test" (#132)
clean up unused/dead comments
fix sql --join-indexes bug when > 2 files joined
support custom sql stmt together with --join-indexes
  • Loading branch information
@liquidaty
liquidaty committed Jul 19, 2023
1 parent c67cd06 commit d56ade5
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 54 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ help:
lib:
@${MAKE} -C src install CONFIGFILE=${CONFIGFILEPATH}

test:
check test:
@${MAKE} -C app test CONFIGFILE=${CONFIGFILEPATH}
@${MAKE} -C examples/lib test CONFIGFILE=${CONFIGFILEPATH}

Expand All @@ -78,4 +78,4 @@ uninstall-app:
uninstall-lib:
${MAKE} -C src uninstall CONFIGFILE=${CONFIGFILEPATH}

.PHONY: help install uninstall uninstall-app uninstall-lib
.PHONY: help install uninstall uninstall-app uninstall-lib check
4 changes: 2 additions & 2 deletions app/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ ${CLEANS}: clean-%:
rm -f ${STANDALONE_PFX}$*${EXE}
rm -f ${BUILD_DIR}/*/*$*.o

.PHONY: all install cli build build-% clean clean-% test test-% lib-%
.PHONY: all install cli build build-% clean clean-% test test-% lib-% check

.SECONDARY: ${OBJECTS}

Expand Down Expand Up @@ -410,7 +410,7 @@ ${YAJL_HELPER_OBJ}: external/yajl_helper/yajl_helper.c
@mkdir -p `dirname "$@"`
${CC} ${CFLAGS} ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -c $< -o $@

test: test-standalone test-cli
check test: test-standalone test-cli

test-standalone:
@${MAKE} -C test test QUIET=1 LEAKS=${LEAKS} CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG}
Expand Down
5 changes: 2 additions & 3 deletions app/compare.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,7 @@ static void zsv_compare_output_begin(struct zsv_compare_data *data) {

// write additional column names
for(struct zsv_compare_added_column *ac = data->added_columns; ac; ac = ac->next)
zsv_compare_header_str(data, ac->colname->name, // ac->colname->name_len,
ZSV_WRITER_SAME_ROW, 1);
zsv_compare_header_str(data, ac->colname->name, ZSV_WRITER_SAME_ROW, 1);

if(data->writer.type == ZSV_COMPARE_OUTPUT_TYPE_JSON && !data->writer.object)
jsonwriter_end_array(data->writer.handle.jsw);
Expand Down Expand Up @@ -591,7 +590,7 @@ static int compare_usage() {
return 0;
}

// TO DO: consolidate common code w sql.c-- move to utils/db.c?
// TO DO: consolidate w sql.c, move common code to utils/db.c
int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts,
const char *opts_used) {
/**
Expand Down
10 changes: 1 addition & 9 deletions app/compare_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ struct zsv_compare_input {

unsigned col_count;
unsigned *out2in; // out2in[output column ix] = input column ix + 1 (zero for no match)
zsv_compare_unique_colname **output_colnames; // colname_ptrs;
zsv_compare_unique_colname **output_colnames;

unsigned key_count;
struct zsv_compare_input_key *keys;
Expand All @@ -69,13 +69,6 @@ struct zsv_compare_added_column {
unsigned col_ix; // index of column in input from which to extract this value
};

/*
struct zsv_compare_sort {
struct zsv_compare_sort *next;
const char *by;
};
*/

struct zsv_compare_data {
enum zsv_compare_status status;
unsigned input_count; // number of allocated compare_input structs
Expand Down Expand Up @@ -108,7 +101,6 @@ struct zsv_compare_data {
struct zsv_opts *opts,
const char *opts_used);

// struct zsv_compare_sort *sort;
sqlite3 *sort_db; // used when --sort option was specified

struct {
Expand Down
28 changes: 1 addition & 27 deletions app/compare_sort.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,31 +26,6 @@ static int zsv_compare_sort_prep_table(struct zsv_compare_data *data,
return rc;
}

/*
static
struct zsv_compare_sort **zsv_compare_sort_add(struct zsv_compare_sort **next,
const char *by,
enum zsv_compare_status *stat) {
struct zsv_compare_sort *e = calloc(1, sizeof(*e));
if(!e)
*stat = zsv_compare_status_memory;
else {
e->by = by;
*next = e;
next = &e->next;
}
return next;
}
static void zsv_compare_sort_delete(struct zsv_compare_sort *sort) {
for(struct zsv_compare_sort *next; sort; sort = next) {
next = sort->next;
free(sort);
}
}
*/

static int zsv_compare_sort_stmt_prep(sqlite3 *db, sqlite3_stmt **stmtp,
// struct zsv_compare_sort *sort,
struct zsv_compare_key *keys,
Expand All @@ -62,7 +37,6 @@ static int zsv_compare_sort_stmt_prep(sqlite3 *db, sqlite3_stmt **stmtp,
}

sqlite3_str_appendf(select_clause, "select * from data%i order by ", ix);
// for(struct zsv_compare_sort *tmp = sort; tmp; tmp = tmp->next)
for(struct zsv_compare_key *key = keys; key; key = key->next)
sqlite3_str_appendf(select_clause, "%s\"%w\"", key == keys ? "" : ", ", key->name);

Expand All @@ -88,7 +62,7 @@ input_init_sorted(struct zsv_compare_data *data,
}
if(rc == SQLITE_OK)
rc = zsv_compare_sort_stmt_prep(data->sort_db, &input->sort_stmt,
data->keys, // data->sort,
data->keys,
input->index);
return rc == SQLITE_OK ? zsv_compare_status_ok : zsv_compare_status_error;
}
Expand Down
57 changes: 46 additions & 11 deletions app/sql.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,23 +33,26 @@ struct string_list {
const char *zsv_sql_usage_msg[] =
{
APPNAME ": run ad hoc sql on a CSV file",
" or join multiple CSV files on one or more common column(s)",
"",
#ifdef NO_STDIN
"Usage: " APPNAME " <filename> [filename ...] <sql | @file.sql>",
#else
"Usage: " APPNAME " [filename, or - for stdin] [filename ...] <sql | @file.sql>",
"Usage: " APPNAME " [filename, or - for stdin] [filename ...] <sql | @file.sql | --join-indexes <N,...>>",
#endif
" e.g. " APPNAME " myfile.csv \"select * from data\"",
" e.g. " APPNAME " myfile.csv myfile2.csv \"select * from data inner join data2\"",
" e.g. " APPNAME " myfile.csv myfile2.csv --join-indexes 1,2",
"",
"Loads your CSV file into a table named 'data', then runs your sql, which must start with 'select '.",
"If multiple files are specified, tables will be named data, data2, data3, ...",
"",
"Options:",
" --join-indexes <n1...>: specify one or more column names to join multiple files by",
" each n is treated as an index in the first input file that determines a column"
" each n is treated as an index in the first input file that determines a column",
" of the join. For example, if joining two files that, respectively, have columns",
" A,B,C,D and X,B,C,A,Y then `--join-indexes 1,3` will join on columns A and C",
" A,B,C,D and X,B,C,A,Y then `--join-indexes 1,3` will join on columns A and C.",
" When using this option, do not include an sql statement",
" -b: output with BOM",
" -C, --max-cols <n> : change the maximum allowable columns. must be > 0 and < 2000",
" -o <output filename> : name of file to save output to",
Expand Down Expand Up @@ -352,10 +355,31 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
// sql template:
// select t1.*, t2.*, t3.* from t1 left join (select * from t2 group by a) t2 left join (select * from t3 group by a) t3 using(a);
sqlite3_stmt *stmt = NULL;
rc = sqlite3_prepare_v2(db, "select * from data", -1, &stmt, NULL);
if(rc != SQLITE_OK)
fprintf(stderr, "%s:\n %s\n (or bad CSV/utf8 input)\n\n", sqlite3_errstr(err), "select * from data");
else {
const char *prefix_search = NULL;
const char *prefix_end = NULL;
if(my_sql) {
prefix_search = " from data ";
prefix_end = strstr(my_sql, prefix_search);
if(!prefix_end) {
prefix_search = " from data";
prefix_end = strstr(my_sql, prefix_search);
if(prefix_end && (prefix_end + strlen(prefix_search) != my_sql + strlen(my_sql)))
prefix_end = NULL;
}
if(!prefix_end || !prefix_search) {
err = 1;
fprintf(stderr, "Invalid sql: must contain 'from data'");
}
}

if(!err) {
rc = sqlite3_prepare_v2(db, "select * from data", -1, &stmt, NULL);
if(rc != SQLITE_OK) {
fprintf(stderr, "%s:\n %s\n (or bad CSV/utf8 input)\n\n", sqlite3_errstr(err), "select * from data");
err = 1;
}
}
if(!err) {
struct string_list **next_joined_column_name = &data.join_column_names;
int col_count = sqlite3_column_count(stmt);
for(char *ix_str = data.join_indexes; !err && ix_str && *ix_str && *(++ix_str); ix_str = strchr(ix_str + 1, ',')) {
Expand Down Expand Up @@ -399,13 +423,24 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
int i = 2;
for(struct string_list *sl = data.more_input_filenames; sl; sl = sl->next, i++) {
sqlite3_str_appendf(select_clause, ", data%i.*", i);
// left join (select * from t2 group by a) t2
// left join (select * from t2 group by a) t2 using(x,...)
sqlite3_str_appendf(from_clause, " left join (select * from data%i group by %s) data%i", i,
sqlite3_str_value(group_by_clause), i);
sqlite3_str_appendf(from_clause, " using (%s)",
sqlite3_str_value(group_by_clause));
}

if(!prefix_end || !prefix_search)
asprintf(&data.sql_dynamic, "select %s from %s",
sqlite3_str_value(select_clause), sqlite3_str_value(from_clause));
else {
asprintf(&data.sql_dynamic, "%.*s from %s%s%s", prefix_end - my_sql, my_sql,
sqlite3_str_value(from_clause),
strlen(prefix_end + strlen(prefix_search)) ? " " : "",
strlen(prefix_end + strlen(prefix_search)) ?
prefix_end + strlen(prefix_search) : "");
}
asprintf(&data.sql_dynamic, "select %s from %s using (%s)",
sqlite3_str_value(select_clause), sqlite3_str_value(from_clause),
sqlite3_str_value(group_by_clause));

my_sql = data.sql_dynamic;
if(opts->verbose)
fprintf(stderr, "Join sql:\n%s\n", my_sql);
Expand Down

0 comments on commit d56ade5

Please sign in to comment.