diff --git a/.travis.yml b/.travis.yml index adaf7d136e7b..1ca9d24fc022 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: - BYOND_MINOR="1427" - NODE_VERSION="4" - RUST_G_VERSION="0.3.0" + - BSQL_VERSION="v1.3.0.2" matrix: include: - env: @@ -37,15 +38,20 @@ matrix: addons: mariadb: '10.2' apt: + sources: + - ubuntu-toolchain-r-test packages: - libstdc++6:i386 - libssl-dev:i386 - gcc-multilib + - g++-7 + - g++-7-multilib + - libmariadbclient-dev:i386 cache: directories: - $HOME/.cargo - $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR} - - $HOME/libmariadb + - $HOME/MariaDB - $HOME/.rustup install: diff --git a/BSQL.dll b/BSQL.dll new file mode 100644 index 000000000000..4e19139307f3 Binary files /dev/null and b/BSQL.dll differ diff --git a/Dockerfile b/Dockerfile index 9fd6f0b6d01b..4d24e938fe38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,20 @@ WORKDIR /rust_g RUN apt-get update && apt-get install -y \ git \ - libssl-dev \ ca-certificates \ + libc6-dev + +FROM build as rust_g + +WORKDIR /rust_g + +RUN apt-get install -y --no-install-recommends \ + libssl-dev \ rustc \ cargo \ - pkg-config \ - && git init \ + pkg-config + +RUN git init \ && git remote add origin https://github.com/tgstation/rust-g #TODO: find a way to read these from .travis.yml or a common source eventually @@ -21,6 +29,38 @@ RUN git fetch --depth 1 origin $RUST_G_VERSION \ && git checkout FETCH_HEAD \ && cargo build --release +FROM base as bsql + +WORKDIR /bsql + +RUN apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + cmake \ + make \ + g++-7 \ + libstdc++6 \ + libmariadb-client-lgpl-dev + +RUN git init \ + && git remote add origin https://github.com/tgstation/BSQL + +#TODO: find a way to read these from .travis.yml or a common source eventually +ENV BSQL_VERSION=v1.3.0.2 + +RUN git fetch --depth 1 origin $BSQL_VERSION \ + && git checkout FETCH_HEAD + +WORKDIR /bsql/artifacts + +ENV CC=gcc-7 CXX=g++-7 + +RUN ln -s /usr/include/mariadb /usr/include/mysql \ + && ln -s /usr/lib/i386-linux-gnu /root/MariaDB \ + && cmake .. \ + && make + FROM base as dm_base WORKDIR /tgstation @@ -64,8 +104,12 @@ RUN apt-get update && apt-get install -y \ && mkdir -p /root/.byond/bin COPY --from=rustg /rust_g/target/release/librust_g.so /root/.byond/bin/rust_g +COPY --from=bsql /bsql/artifacts/src/BSQL/libBSQL.so ./ COPY --from=build /deploy ./ +#bsql fexists memes +RUN ln -s /tgstation/libBSQL.so /root/.byond/bin/libBSQL.so + VOLUME [ "/tgstation/config", "/tgstation/data" ] ENTRYPOINT [ "DreamDaemon", "tgstation.dmb", "-port", "1337", "-trusted", "-close", "-verbose" ] diff --git a/code/__DEFINES/_protect.dm b/code/__DEFINES/_protect.dm new file mode 100644 index 000000000000..f710ca77e355 --- /dev/null +++ b/code/__DEFINES/_protect.dm @@ -0,0 +1,10 @@ +#define GENERAL_PROTECT_DATUM(Path)\ +##Path/can_vv_get(var_name){\ + return FALSE;\ +}\ +##Path/vv_edit_var(var_name, var_value){\ + return FALSE;\ +}\ +##Path/CanProcCall(procname){\ + return FALSE;\ +} \ No newline at end of file diff --git a/code/__DEFINES/bsql.config.dm b/code/__DEFINES/bsql.config.dm new file mode 100644 index 000000000000..ce1964c217cc --- /dev/null +++ b/code/__DEFINES/bsql.config.dm @@ -0,0 +1,6 @@ +#define BSQL_EXTERNAL_CONFIGURATION +#define BSQL_DEL_PROC(path) ##path/Destroy() +#define BSQL_DEL_CALL(obj) qdel(##obj) +#define BSQL_IS_DELETED(obj) (QDELETED(obj)) +#define BSQL_PROTECT_DATUM(path) GENERAL_PROTECT_DATUM(##path) +#define BSQL_ERROR(message) SSdbcore.ReportError(message) diff --git a/code/__DEFINES/bsql.dm b/code/__DEFINES/bsql.dm new file mode 100644 index 000000000000..cd15115c7d46 --- /dev/null +++ b/code/__DEFINES/bsql.dm @@ -0,0 +1,132 @@ +//BSQL - DMAPI v1.2.0.1 + +//types of connections +#define BSQL_CONNECTION_TYPE_MARIADB "MySql" +#define BSQL_CONNECTION_TYPE_SQLSERVER "SqlServer" + +#define BSQL_DEFAULT_TIMEOUT 5 + +//Call this before rebooting or shutting down your world to clean up gracefully. This invalidates all active connection and operation datums +/world/proc/BSQL_Shutdown() + return + +/* +Called whenever a library call is made with verbose information, override and do with as you please + message: English debug message +*/ +/world/proc/BSQL_Debug(msg) + return + +/* +Create a new database connection, does not perform the actual connect + connection_type: The BSQL connection_type to use + asyncTimeout: The timeout to use for normal operations, 0 for infinite, defaults to BSQL_DEFAULT_TIMEOUT + blockingTimeout: The timeout to use for blocking operations, must be less than or equal to asyncTimeout, 0 for infinite, defaults to asyncTimeout +*/ +/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout) + return ..() + +/* +Starts an operation to connect to a database. Should only have 1 successful call + ipaddress: The ip/hostname of the target server + port: The port of the target server + username: The username to login to the target server + password: The password for the target server + database: Optional database to connect to. Must be used when trying to do database operations, `USE x` is not sufficient + Returns: A /datum/BSQL_Operation representing the connection or null if an error occurred +*/ +/datum/BSQL_Connection/proc/BeginConnect(ipaddress, port, username, password, database) + return + +/* +Properly quotes a string for use by the database. The connection must be open for this proc to succeed + str: The string to quote + Returns: The string quoted on success, null on error +*/ +/datum/BSQL_Connection/proc/Quote(str) + return + +/* +Starts an operation for a query + query: The text of the query. Only one query allowed per invocation, no semicolons + Returns: A /datum/BSQL_Operation/Query representing the running query and subsequent result set or null if an error occurred + + Note for MariaDB: The underlying connection is pooled. In order to use connection state based properties (i.e. LAST_INSERT_ID()) you can guarantee multiple queries will use the same connection by running BSQL_DEL_CALL(query) on the finished /datum/BSQL_Operation/Query and then creating the next one with another call to BeginQuery() with no sleeps in between +*/ +/datum/BSQL_Connection/proc/BeginQuery(query) + return + +/* +Checks if the operation is complete. This, in some cases must be called multiple times with false return before a result is present regardless of timespan. For best performance check it once per tick + + Returns: TRUE if the operation is complete, FALSE if it's not, null on error +*/ +/datum/BSQL_Operation/proc/IsComplete() + return + +/* +Blocks the entire game until the given operation completes. IsComplete should not be checked after calling this to avoid potential side effects. + +Returns: TRUE on success, FALSE if the operation wait time exceeded the connection's blockingTimeout setting +*/ +/datum/BSQL_Operation/proc/WaitForCompletion() + return + +/* +Get the error message associated with an operation. Should not be used while IsComplete() returns FALSE + + Returns: The error message, if any. null otherwise +*/ +/datum/BSQL_Operation/proc/GetError() + return + +/* +Get the error code associated with an operation. Should not be used while IsComplete() returns FALSE + + Returns: The error code, if any. null otherwise +*/ +/datum/BSQL_Operation/proc/GetErrorCode() + return + +/* +Gets an associated list of column name -> value representation of the most recent row in the query. Only valid if IsComplete() returns TRUE. If this returns null and no errors are present there are no more results in the query. Important to note that once IsComplete() returns TRUE it must not be called again without checking this or the row values may be lost + + Returns: An associated list of column name -> value for the row. Values will always be either strings or null +*/ +/datum/BSQL_Operation/Query/proc/CurrentRow() + return + + +/* +Code configuration options below + +Define this to avoid modifying this file but the following defines must be declared somewhere else before BSQL/includes.dm is included +*/ +#ifndef BSQL_EXTERNAL_CONFIGURATION + +//Modify this if you disagree with byond's GC schemes. Ensure this is called for all connections and operations when they are deleted or they will leak native resources until /world/proc/BSQL_Shutdown() is called +#define BSQL_DEL_PROC(path) ##path/Del() + +//The equivalent of calling del() in your codebase +#define BSQL_DEL_CALL(obj) del(##obj) + +//Returns TRUE if an object is delete +#define BSQL_IS_DELETED(obj) (obj == null) + +//Modify this to add protections to the connection and query datums +#define BSQL_PROTECT_DATUM(path) + +//Modify this to change up error handling for the library +#define BSQL_ERROR(message) CRASH("BSQL: [##message]") + +#endif + +/* +Copyright 2018 Jordan Brown + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ diff --git a/code/__DEFINES/tgs.config.dm b/code/__DEFINES/tgs.config.dm index 00cbae7629af..b05cff7e010a 100644 --- a/code/__DEFINES/tgs.config.dm +++ b/code/__DEFINES/tgs.config.dm @@ -7,13 +7,4 @@ #define TGS_ERROR_LOG(message) log_world("TGS: Error: [##message]") #define TGS_NOTIFY_ADMINS(event) message_admins(##event) #define TGS_CLIENT_COUNT GLOB.clients.len -#define TGS_PROTECT_DATUM(Path)\ -##Path/can_vv_get(var_name){\ - return FALSE;\ -}\ -##Path/vv_edit_var(var_name, var_value){\ - return FALSE;\ -}\ -##Path/CanProcCall(procname){\ - return FALSE;\ -} +#define TGS_PROTECT_DATUM(Path) GENERAL_PROTECT_DATUM(##Path) diff --git a/code/__HELPERS/text.dm b/code/__HELPERS/text.dm index 2db7b8bc7828..7d6c017b7736 100644 --- a/code/__HELPERS/text.dm +++ b/code/__HELPERS/text.dm @@ -15,8 +15,7 @@ // Run all strings to be used in an SQL query through this proc first to properly escape out injection attempts. /proc/sanitizeSQL(t) - var/sqltext = SSdbcore.Quote("[t]"); - return copytext(sqltext, 2, lentext(sqltext));//Quote() adds quotes around input, we already do that + return SSdbcore.Quote("[t]") /proc/format_table_name(table as text) return CONFIG_GET(string/feedback_tableprefix) + table diff --git a/code/controllers/configuration/entries/dbconfig.dm b/code/controllers/configuration/entries/dbconfig.dm index 1eb1186a8b7f..72f190c2f17b 100644 --- a/code/controllers/configuration/entries/dbconfig.dm +++ b/code/controllers/configuration/entries/dbconfig.dm @@ -28,4 +28,20 @@ /datum/config_entry/number/query_debug_log_timeout config_entry_value = 70 min_val = 1 - protection = CONFIG_ENTRY_LOCKED | CONFIG_ENTRY_HIDDEN + protection = CONFIG_ENTRY_LOCKED + deprecated_by = /datum/config_entry/number/blocking_query_timeout + +/datum/config_entry/number/query_debug_log_timeout/DeprecationUpdate(value) + return value + +/datum/config_entry/number/async_query_timeout + config_entry_value = 10 + min_val = 0 + protection = CONFIG_ENTRY_LOCKED + +/datum/config_entry/number/blocking_query_timeout + config_entry_value = 5 + min_val = 0 + protection = CONFIG_ENTRY_LOCKED + +/datum/config_entry/flag/bsql_debug diff --git a/code/controllers/subsystem/dbcore.dm b/code/controllers/subsystem/dbcore.dm index 34bf57594dd2..fab7db7e629c 100644 --- a/code/controllers/subsystem/dbcore.dm +++ b/code/controllers/subsystem/dbcore.dm @@ -5,32 +5,16 @@ SUBSYSTEM_DEF(dbcore) init_order = INIT_ORDER_DBCORE var/const/FAILED_DB_CONNECTION_CUTOFF = 5 - var/const/Default_Cursor = 0 - var/const/Client_Cursor = 1 - var/const/Server_Cursor = 2 - //conversions - var/const/TEXT_CONV = 1 - var/const/RSC_FILE_CONV = 2 - var/const/NUMBER_CONV = 3 - //column flag values: - var/const/IS_NUMERIC = 1 - var/const/IS_BINARY = 2 - var/const/IS_NOT_NULL = 4 - var/const/IS_PRIMARY_KEY = 8 - var/const/IS_UNSIGNED = 16 var/schema_mismatch = 0 var/db_minor = 0 var/db_major = 0 -// TODO: Investigate more recent type additions and see if I can handle them. - Nadrew - - var/_db_con// This variable contains a reference to the actual database connection. var/failed_connections = 0 + var/last_error var/list/active_queries = list() -/datum/controller/subsystem/dbcore/PreInit() - if(!_db_con) - _db_con = _dm_db_new_con() + var/datum/BSQL_Connection/connection + var/datum/BSQL_Operation/connectOperation /datum/controller/subsystem/dbcore/Initialize() //We send warnings to the admins during subsystem init, as the clients will be New'd and messages @@ -54,7 +38,8 @@ SUBSYSTEM_DEF(dbcore) return /datum/controller/subsystem/dbcore/Recover() - _db_con = SSdbcore._db_con + connection = SSdbcore.connection + connectOperation = SSdbcore.connectOperation /datum/controller/subsystem/dbcore/Shutdown() //This is as close as we can get to the true round end before Disconnect() without changing where it's called, defeating the reason this is a subsystem @@ -64,13 +49,14 @@ SUBSYSTEM_DEF(dbcore) qdel(query_round_shutdown) if(IsConnected()) Disconnect() + world.BSQL_Shutdown() //nu /datum/controller/subsystem/dbcore/can_vv_get(var_name) - return var_name != NAMEOF(src, _db_con) && var_name != NAMEOF(src, active_queries) && ..() + return var_name != NAMEOF(src, connection) && var_name != NAMEOF(src, active_queries) && var_name != NAMEOF(src, connectOperation) && ..() /datum/controller/subsystem/dbcore/vv_edit_var(var_name, var_value) - if(var_name == "_db_con") + if(var_name == NAMEOF(src, connection) || var_name == NAMEOF(src, connectOperation)) return FALSE return ..() @@ -90,17 +76,30 @@ SUBSYSTEM_DEF(dbcore) var/address = CONFIG_GET(string/address) var/port = CONFIG_GET(number/port) - _dm_db_connect(_db_con, "dbi:mysql:[db]:[address]:[port]", user, pass, Default_Cursor, null) - . = IsConnected() + connection = new /datum/BSQL_Connection(BSQL_CONNECTION_TYPE_MARIADB, CONFIG_GET(number/async_query_timeout), CONFIG_GET(number/blocking_query_timeout)) + var/error + if(QDELETED(connection)) + connection = null + error = last_error + else + SSdbcore.last_error = null + connectOperation = connection.BeginConnect(address, port, user, pass, db) + if(SSdbcore.last_error) + CRASH(SSdbcore.last_error) + UNTIL(connectOperation.IsComplete()) + error = connectOperation.GetError() + . = !error if (!.) - log_sql("Connect() failed | [ErrorMsg()]") + log_sql("Connect() failed | [error]") ++failed_connections + QDEL_NULL(connection) + QDEL_NULL(connectOperation) /datum/controller/subsystem/dbcore/proc/CheckSchemaVersion() if(CONFIG_GET(flag/sql_enabled)) - if(SSdbcore.Connect()) + if(Connect()) log_world("Database connection established.") - var/datum/DBQuery/query_db_version = SSdbcore.NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1") + var/datum/DBQuery/query_db_version = NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1") query_db_version.Execute() if(query_db_version.NextRow()) db_major = text2num(query_db_version.item[1]) @@ -146,27 +145,36 @@ SUBSYSTEM_DEF(dbcore) /datum/controller/subsystem/dbcore/proc/Disconnect() failed_connections = 0 - return _dm_db_close(_db_con) + QDEL_NULL(connectOperation) + QDEL_NULL(connection) /datum/controller/subsystem/dbcore/proc/IsConnected() if(!CONFIG_GET(flag/sql_enabled)) return FALSE - return _dm_db_is_connected(_db_con) + //block until any connect operations finish + var/datum/BSQL_Connection/_connection = connection + var/datum/BSQL_Operation/op = connectOperation + UNTIL(QDELETED(_connection) || op.IsComplete()) + return !QDELETED(connection) && !op.GetError() /datum/controller/subsystem/dbcore/proc/Quote(str) - return _dm_db_quote(_db_con, str) + if(connection) + return connection.Quote(str) /datum/controller/subsystem/dbcore/proc/ErrorMsg() if(!CONFIG_GET(flag/sql_enabled)) return "Database disabled by configuration" - return _dm_db_error_msg(_db_con) + return last_error + +/datum/controller/subsystem/dbcore/proc/ReportError(error) + last_error = error -/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query, cursor_handler = Default_Cursor) +/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query) if(IsAdminAdvancedProcCall()) log_admin_private("ERROR: Advanced admin proc call led to sql query: [sql_query]. Query has been blocked") message_admins("ERROR: Advanced admin proc call led to sql query. Query has been blocked") return FALSE - return new /datum/DBQuery(sql_query, src, cursor_handler) + return new /datum/DBQuery(sql_query, connection) /* Takes a list of rows (each row being an associated list of column => value) and inserts them via a single mass query. @@ -180,7 +188,7 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table It was included because it is still supported in mariadb. It does not work with duplicate_key and the mysql server ignores it in those cases */ -/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE) +/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE, async = FALSE) if (!table || !rows || !istype(rows)) return var/list/columns = list() @@ -230,34 +238,30 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table sqlrowlist = " [sqlrowlist.Join(",\n ")]" var/datum/DBQuery/Query = NewQuery("INSERT[delayed][ignore_errors] INTO [table]\n([columns.Join(", ")])\nVALUES\n[sqlrowlist]\n[duplicate_key]") if (warn) - . = Query.warn_execute() + . = Query.warn_execute(async) else - . = Query.Execute() + . = Query.Execute(async) qdel(Query) - /datum/DBQuery var/sql // The sql query being executed. - var/default_cursor - var/list/columns //list of DB Columns populated by Columns() - var/list/conversions var/list/item //list of data values populated by NextRow() + var/last_activity var/last_activity_time - var/datum/controller/subsystem/dbcore/db_connection - var/_db_query -/datum/DBQuery/New(sql_query, datum/controller/subsystem/dbcore/connection_handler, cursor_handler) + var/last_error + var/skip_next_is_complete + var/in_progress + var/datum/BSQL_Connection/connection + var/datum/BSQL_Operation/Query/query + +/datum/DBQuery/New(sql_query, datum/BSQL_Connection/connection) SSdbcore.active_queries[src] = TRUE Activity("Created") - if(sql_query) - sql = sql_query - if(connection_handler) - db_connection = connection_handler - if(cursor_handler) - default_cursor = cursor_handler item = list() - _db_query = _dm_db_new_query() + src.connection = connection + sql = sql_query /datum/DBQuery/Destroy() Close() @@ -268,147 +272,93 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table //fuck off kevinz return FALSE +/datum/DBQuery/proc/SetQuery(new_sql) + if(in_progress) + CRASH("Attempted to set new sql while waiting on active query") + Close() + sql = new_sql + /datum/DBQuery/proc/Activity(activity) last_activity = activity last_activity_time = world.time -/datum/DBQuery/proc/warn_execute() - . = Execute() +/datum/DBQuery/proc/warn_execute(async = FALSE) + . = Execute(async) if(!.) to_chat(usr, "A SQL error occurred during this operation, check the server logs.") -/datum/DBQuery/proc/SetQuery(new_sql) - Activity("SetQuery") - Close() - sql = new_sql - -/datum/DBQuery/proc/Execute(sql_query = sql, cursor_handler = default_cursor, log_error = TRUE) +/datum/DBQuery/proc/Execute(async = FALSE, log_error = TRUE) Activity("Execute") + if(in_progress) + CRASH("Attempted to start a new query while waiting on the old one") + + if(QDELETED(connection)) + last_error = "No connection!" + return FALSE + var/start_time - var/timeout = CONFIG_GET(number/query_debug_log_timeout) - if(timeout) + var/timed_out + if(!async) start_time = REALTIMEOFDAY Close() - . = _dm_db_execute(_db_query, sql_query, db_connection._db_con, cursor_handler, null) + query = connection.BeginQuery(sql) + if(!async) + timed_out = !query.WaitForCompletion() + else + in_progress = TRUE + UNTIL(query.IsComplete()) + in_progress = FALSE + skip_next_is_complete = TRUE + var/error = QDELETED(query) ? "Query object deleted!" : query.GetError() + last_error = error + . = !error if(!. && log_error) - log_sql("[ErrorMsg()] | Query used: [sql]") - if(timeout) - if((REALTIMEOFDAY - start_time) > timeout) - log_query_debug("Query execution started at [start_time]") - log_query_debug("Query execution ended at [REALTIMEOFDAY]") - log_query_debug("Possible slow query timeout detected.") - log_query_debug("Query used: [sql]") - slow_query_check() + log_sql("[error] | Query used: [sql]") + if(!async && timed_out) + log_query_debug("Query execution started at [start_time]") + log_query_debug("Query execution ended at [REALTIMEOFDAY]") + log_query_debug("Slow query timeout detected.") + log_query_debug("Query used: [sql]") + slow_query_check() /datum/DBQuery/proc/slow_query_check() - message_admins("HEY! A database query may have timed out. Did the server just hang? \[YES\]|\[NO\]") + message_admins("HEY! A database query timed out. Did the server just hang? \[YES\]|\[NO\]") -/datum/DBQuery/proc/NextRow() +/datum/DBQuery/proc/NextRow(async) Activity("NextRow") - return _dm_db_next_row(_db_query,item,conversions) + UNTIL(!in_progress) + if(!skip_next_is_complete) + if(!async) + query.WaitForCompletion() + else + in_progress = TRUE + UNTIL(query.IsComplete()) + in_progress = FALSE + else + skip_next_is_complete = FALSE -/datum/DBQuery/proc/RowsAffected() - return _dm_db_rows_affected(_db_query) + last_error = query.GetError() + var/list/results = query.CurrentRow() + . = results != null -/datum/DBQuery/proc/RowCount() - return _dm_db_row_count(_db_query) + item.Cut() + //populate item array + for(var/I in results) + item += results[I] /datum/DBQuery/proc/ErrorMsg() - return _dm_db_error_msg(_db_query) - -/datum/DBQuery/proc/Columns() - if(!columns) - columns = _dm_db_columns(_db_query, /datum/DBColumn) - return columns - -/datum/DBQuery/proc/GetRowData() - var/list/columns = Columns() - var/list/results - if(columns.len) - results = list() - for(var/C in columns) - results+=C - var/datum/DBColumn/cur_col = columns[C] - results[C] = src.item[(cur_col.position+1)] - return results + return last_error /datum/DBQuery/proc/Close() item.Cut() - columns = null - conversions = null - return _dm_db_close(_db_query) - -/datum/DBQuery/proc/Quote(str) - return db_connection.Quote(str) - -/datum/DBQuery/proc/SetConversion(column,conversion) - if(istext(column)) - column = columns.Find(column) - if(!conversions) - conversions = new /list(column) - else if(conversions.len < column) - conversions.len = column - conversions[column] = conversion - - -/datum/DBColumn - var/name - var/table - var/position //1-based index into item data - var/sql_type - var/flags - var/length - var/max_length - //types - var/const/TINYINT = 1 - var/const/SMALLINT = 2 - var/const/MEDIUMINT = 3 - var/const/INTEGER = 4 - var/const/BIGINT = 5 - var/const/DECIMAL = 6 - var/const/FLOAT = 7 - var/const/DOUBLE = 8 - var/const/DATE = 9 - var/const/DATETIME = 10 - var/const/TIMESTAMP = 11 - var/const/TIME = 12 - var/const/STRING = 13 - var/const/BLOB = 14 - -/datum/DBColumn/New(name_handler, table_handler, position_handler, type_handler, flag_handler, length_handler, max_length_handler) - name = name_handler - table = table_handler - position = position_handler - sql_type = type_handler - flags = flag_handler - length = length_handler - max_length = max_length_handler - -/datum/DBColumn/proc/SqlTypeName(type_handler = sql_type) - switch(type_handler) - if(TINYINT) - return "TINYINT" - if(SMALLINT) - return "SMALLINT" - if(MEDIUMINT) - return "MEDIUMINT" - if(INTEGER) - return "INTEGER" - if(BIGINT) - return "BIGINT" - if(FLOAT) - return "FLOAT" - if(DOUBLE) - return "DOUBLE" - if(DATE) - return "DATE" - if(DATETIME) - return "DATETIME" - if(TIMESTAMP) - return "TIMESTAMP" - if(TIME) - return "TIME" - if(STRING) - return "STRING" - if(BLOB) - return "BLOB" + QDEL_NULL(query) + +/world/BSQL_Debug(message) + if(!CONFIG_GET(flag/bsql_debug)) + return + + //strip sensitive stuff + if(findtext(message, ": CreateConnection(")) + message = "CreateConnection CENSORED" + + log_sql("BSQL_DEBUG: [message]") diff --git a/code/modules/bsql/LICENSE b/code/modules/bsql/LICENSE new file mode 100644 index 000000000000..882f6d457160 --- /dev/null +++ b/code/modules/bsql/LICENSE @@ -0,0 +1,7 @@ +Copyright 2018 Jordan Brown + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/code/modules/bsql/core/connection.dm b/code/modules/bsql/core/connection.dm new file mode 100644 index 000000000000..394255787292 --- /dev/null +++ b/code/modules/bsql/core/connection.dm @@ -0,0 +1,66 @@ +/datum/BSQL_Connection + var/id + var/connection_type + +BSQL_PROTECT_DATUM(/datum/BSQL_Connection) + +/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout) + if(asyncTimeout == null) + asyncTimeout = BSQL_DEFAULT_TIMEOUT + if(blockingTimeout == null) + blockingTimeout = asyncTimeout + + src.connection_type = connection_type + + world._BSQL_InitCheck(src) + + var/error = world._BSQL_Internal_Call("CreateConnection", connection_type, "[asyncTimeout]", "[blockingTimeout]") + if(error) + BSQL_ERROR(error) + return + + id = world._BSQL_Internal_Call("GetConnection") + if(!id) + BSQL_ERROR("BSQL library failed to provide connect operation for connection id [id]([connection_type])!") + +BSQL_DEL_PROC(/datum/BSQL_Connection) + var/error + if(id) + error = world._BSQL_Internal_Call("ReleaseConnection", id) + . = ..() + if(error) + BSQL_ERROR(error) + +/datum/BSQL_Connection/BeginConnect(ipaddress, port, username, password, database) + var/error = world._BSQL_Internal_Call("OpenConnection", id, ipaddress, "[port]", username, password, database) + if(error) + BSQL_ERROR(error) + return + + var/op_id = world._BSQL_Internal_Call("GetOperation") + if(!op_id) + BSQL_ERROR("Library failed to provide connect operation for connection id [id]([connection_type])!") + return + + return new /datum/BSQL_Operation(src, op_id) + + +/datum/BSQL_Connection/BeginQuery(query) + var/error = world._BSQL_Internal_Call("NewQuery", id, query) + if(error) + BSQL_ERROR(error) + return + + var/op_id = world._BSQL_Internal_Call("GetOperation") + if(!op_id) + BSQL_ERROR("Library failed to provide query operation for connection id [id]([connection_type])!") + return + + return new /datum/BSQL_Operation/Query(src, op_id) + +/datum/BSQL_Connection/Quote(str) + if(!str) + return null; + . = world._BSQL_Internal_Call("QuoteString", id, "[str]") + if(!.) + BSQL_ERROR("Library failed to provide quote for [str]!") \ No newline at end of file diff --git a/code/modules/bsql/core/library.dm b/code/modules/bsql/core/library.dm new file mode 100644 index 000000000000..b8e764a16d93 --- /dev/null +++ b/code/modules/bsql/core/library.dm @@ -0,0 +1,37 @@ +/world/proc/_BSQL_Internal_Call(func, ...) + var/list/call_args = args.Copy(2) + BSQL_Debug("[.....]: [args[1]]([call_args.Join(", ")])") + . = call(_BSQL_Library_Path(), func)(arglist(call_args)) + BSQL_Debug("Result: [. == null ? "NULL" : "\"[.]\""]") + +/world/proc/_BSQL_Library_Path() + return system_type == MS_WINDOWS ? "BSQL.dll" : "libBSQL.so" + +/world/proc/_BSQL_InitCheck(datum/BSQL_Connection/caller) + var/static/library_initialized = FALSE + if(_BSQL_Initialized()) + return + var/libPath = _BSQL_Library_Path() + if(!fexists(libPath)) + BSQL_DEL_CALL(caller) + BSQL_ERROR("Could not find [libPath]!") + return + + var/result = _BSQL_Internal_Call("Initialize") + if(result) + BSQL_DEL_CALL(caller) + BSQL_ERROR(result) + return + _BSQL_Initialized(TRUE) + +/world/proc/_BSQL_Initialized(new_val) + var/static/bsql_library_initialized = FALSE + if(new_val != null) + bsql_library_initialized = new_val + return bsql_library_initialized + +/world/BSQL_Shutdown() + if(!_BSQL_Initialized()) + return + _BSQL_Internal_Call("Shutdown") + _BSQL_Initialized(FALSE) diff --git a/code/modules/bsql/core/operation.dm b/code/modules/bsql/core/operation.dm new file mode 100644 index 000000000000..50dce6ae5f64 --- /dev/null +++ b/code/modules/bsql/core/operation.dm @@ -0,0 +1,47 @@ +/datum/BSQL_Operation + var/datum/BSQL_Connection/connection + var/id + +BSQL_PROTECT_DATUM(/datum/BSQL_Operation) + +/datum/BSQL_Operation/New(datum/BSQL_Connection/connection, id) + src.connection = connection + src.id = id + +BSQL_DEL_PROC(/datum/BSQL_Operation) + var/error + if(!BSQL_IS_DELETED(connection)) + error = world._BSQL_Internal_Call("ReleaseOperation", connection.id, id) + . = ..() + if(error) + BSQL_ERROR(error) + +/datum/BSQL_Operation/IsComplete() + if(BSQL_IS_DELETED(connection)) + return TRUE + var/result = world._BSQL_Internal_Call("OpComplete", connection.id, id) + if(!result) + BSQL_ERROR("Error fetching operation [id] for connection [connection.id]!") + return + return result == "DONE" + +/datum/BSQL_Operation/GetError() + if(BSQL_IS_DELETED(connection)) + return "Connection deleted!" + return world._BSQL_Internal_Call("GetError", connection.id, id) + +/datum/BSQL_Operation/GetErrorCode() + if(BSQL_IS_DELETED(connection)) + return -2 + return text2num(world._BSQL_Internal_Call("GetErrorCode", connection.id, id)) + +/datum/BSQL_Operation/WaitForCompletion() + if(BSQL_IS_DELETED(connection)) + return + var/error = world._BSQL_Internal_Call("BlockOnOperation", connection.id, id) + if(error) + if(error == "Operation timed out!") //match this with the implementation + return FALSE + BSQL_ERROR("Error waiting for operation [id] for connection [connection.id]! [error]") + return + return TRUE diff --git a/code/modules/bsql/core/query.dm b/code/modules/bsql/core/query.dm new file mode 100644 index 000000000000..96c3714c7156 --- /dev/null +++ b/code/modules/bsql/core/query.dm @@ -0,0 +1,35 @@ +/datum/BSQL_Operation/Query + var/last_result_json + var/list/last_result + +BSQL_PROTECT_DATUM(/datum/BSQL_Operation/Query) + +/datum/BSQL_Operation/Query/CurrentRow() + return last_result + +/datum/BSQL_Operation/Query/IsComplete() + //whole different ballgame here + if(BSQL_IS_DELETED(connection)) + return TRUE + var/result = world._BSQL_Internal_Call("ReadyRow", connection.id, id) + switch(result) + if("DONE") + //load the data + LoadQueryResult() + return TRUE + if("NOTDONE") + return FALSE + else + BSQL_ERROR(result) + +/datum/BSQL_Operation/Query/WaitForCompletion() + . = ..() + if(.) + LoadQueryResult() + +/datum/BSQL_Operation/Query/proc/LoadQueryResult() + last_result_json = world._BSQL_Internal_Call("GetRow", connection.id, id) + if(last_result_json) + last_result = json_decode(last_result_json) + else + last_result = null diff --git a/code/modules/bsql/includes.dm b/code/modules/bsql/includes.dm new file mode 100644 index 000000000000..ec199a5513a6 --- /dev/null +++ b/code/modules/bsql/includes.dm @@ -0,0 +1,4 @@ +#include "core\connection.dm" +#include "core\library.dm" +#include "core\operation.dm" +#include "core\query.dm" diff --git a/config/dbconfig.txt b/config/dbconfig.txt index ed0ffadede84..65ab7ed6735a 100644 --- a/config/dbconfig.txt +++ b/config/dbconfig.txt @@ -29,6 +29,14 @@ FEEDBACK_LOGIN username ## Password used to access the database. FEEDBACK_PASSWORD password -## Time in deciseconds for a query to execute before alerting a for possible slow query timeout. -## While enabled queries and their execution times are logged if they exceed this value. -#QUERY_DEBUG_LOG_TIMEOUT 70 +## Time in seconds for asynchronous queries to timeout +## Set to 0 for infinite +ASYNC_QUERY_TIMEOUT 10 + +## Time in seconds for blocking queries to execute before alerting a for possible slow query timeout +## Set to 0 for infinite +## Must be less than or equal to ASYNC_QUERY_TIMEOUT +BLOCKING_QUERY_TIMEOUT 5 + +## Uncomment to enable verbose BSQL communication logs +#BSQL_DEBUG diff --git a/libmariadb.dll b/libmariadb.dll index f72c9509c3ef..2e185d265761 100644 Binary files a/libmariadb.dll and b/libmariadb.dll differ diff --git a/tgstation.dme b/tgstation.dme index 5323fab08557..3fb9016d2654 100644 --- a/tgstation.dme +++ b/tgstation.dme @@ -17,8 +17,11 @@ #include "code\_compile_options.dm" #include "code\world.dm" #include "code\__DEFINES\_globals.dm" +#include "code\__DEFINES\_protect.dm" #include "code\__DEFINES\_tick.dm" #include "code\__DEFINES\access.dm" +#include "code\__DEFINES\bsql.config.dm" +#include "code\__DEFINES\bsql.dm" #include "code\__DEFINES\admin.dm" #include "code\__DEFINES\antagonists.dm" #include "code\__DEFINES\atmospherics.dm" @@ -1346,6 +1349,7 @@ #include "code\modules\awaymissions\mission_code\stationCollision.dm" #include "code\modules\awaymissions\mission_code\undergroundoutpost45.dm" #include "code\modules\awaymissions\mission_code\wildwest.dm" +#include "code\modules\bsql\includes.dm" #include "code\modules\cargo\bounty.dm" #include "code\modules\cargo\bounty_console.dm" #include "code\modules\cargo\console.dm" diff --git a/tools/travis/build_byond.sh b/tools/travis/build_byond.sh index 7d8e7d9f61f5..584ac55756dd 100755 --- a/tools/travis/build_byond.sh +++ b/tools/travis/build_byond.sh @@ -52,20 +52,6 @@ if [ "$BUILD_TOOLS" = false ]; then #test config cp tools/travis/travis_config.txt config/config.txt - - # get libmariadb, cache it so limmex doesn't get angery - if [ -f $HOME/libmariadb ]; then - #travis likes to interpret the cache command as it being a file for some reason - rm $HOME/libmariadb - mkdir $HOME/libmariadb - fi - if [ ! -f $HOME/libmariadb/libmariadb.so ]; then - wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz - tar -xvf mariadb_client-2.0.0-linux.tgz - mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so - rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux - fi - ln -s $HOME/libmariadb/libmariadb.so libmariadb.so DreamDaemon yogstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis" cat data/logs/travis/clean_run.lk diff --git a/tools/travis/build_dependencies.sh b/tools/travis/build_dependencies.sh index 5884423d69db..8de5ffa7db30 100755 --- a/tools/travis/build_dependencies.sh +++ b/tools/travis/build_dependencies.sh @@ -16,4 +16,39 @@ if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then mkdir -p ~/.byond/bin ln -s $PWD/target/release/librust_g.so ~/.byond/bin/rust_g + + mkdir -p ../BSQL/artifacts + cd ../BSQL + git init + git remote add origin https://github.com/tgstation/BSQL + git fetch --depth 1 origin $BSQL_VERSION + git checkout FETCH_HEAD + + if [ -f "$HOME/MariaDB/libmariadb.so.2" ] && [ -f "$HOME/MariaDB/libmariadb.so" ] && [ -d "$HOME/MariaDB/include" ]; + then + echo "Using cached MariaDB library." + else + echo "Setting up MariaDB." + rm -rf "$HOME/MariaDB" + mkdir -p "$HOME/MariaDB" + wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-client-lgpl/libmariadb2_2.0.0-1_i386.deb + dpkg -x libmariadb2_2.0.0-1_i386.deb /tmp/extract + rm libmariadb2_2.0.0-1_i386.deb + mv /tmp/extract/usr/lib/i386-linux-gnu/libmariadb.so.2 $HOME/MariaDB/ + ln -s $HOME/MariaDB/libmariadb.so.2 $HOME/MariaDB/libmariadb.so + rm -rf /tmp/extract + + wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-connector-c/libmariadb-dev_2.3.3-1_i386.deb + dpkg -x libmariadb-dev_2.3.3-1_i386.deb /tmp/extract + rm libmariadb-dev_2.3.3-1_i386.deb + mv /tmp/extract/usr/include $HOME/MariaDB/ + #fuck what is this even? + mv $HOME/MariaDB/include/mariadb $HOME/MariaDB/include/mysql + fi + + cd artifacts + export CXX=g++-7 + cmake .. -DMARIA_INCLUDE_DIR=$HOME/MariaDB/include + make + mv src/BSQL/libBSQL.so ../../ fi