From e987b391a1c643d4eb727470b49981cd0ef259fc Mon Sep 17 00:00:00 2001 From: Michael du Breuil Date: Wed, 5 Feb 2020 15:58:33 -0700 Subject: [PATCH 1/3] AP_Scripting: Add support for REPL over MAVLink --- libraries/AP_Scripting/AP_Scripting.cpp | 51 ++++- libraries/AP_Scripting/AP_Scripting.h | 14 ++ libraries/AP_Scripting/lua_repl.cpp | 261 ++++++++++++++++++++++++ libraries/AP_Scripting/lua_scripts.cpp | 57 ++++-- libraries/AP_Scripting/lua_scripts.h | 39 +++- 5 files changed, 406 insertions(+), 16 deletions(-) create mode 100644 libraries/AP_Scripting/lua_repl.cpp diff --git a/libraries/AP_Scripting/AP_Scripting.cpp b/libraries/AP_Scripting/AP_Scripting.cpp index cd12781af08e8..7dbe7ff98d1d9 100644 --- a/libraries/AP_Scripting/AP_Scripting.cpp +++ b/libraries/AP_Scripting/AP_Scripting.cpp @@ -104,8 +104,57 @@ void AP_Scripting::init(void) { } } +MAV_RESULT AP_Scripting::handle_command_int_packet(const mavlink_command_int_t &packet) { + switch ((SCRIPTING_CMD)packet.param1) { + case SCRIPTING_CMD_REPL_START: + return repl_start() ? MAV_RESULT_ACCEPTED : MAV_RESULT_FAILED; + case SCRIPTING_CMD_REPL_STOP: + repl_stop(); + return MAV_RESULT_ACCEPTED; + case SCRIPTING_CMD_ENUM_END: // cope with MAVLink generator appending to our enum + break; + } + + return MAV_RESULT_UNSUPPORTED; +} + +bool AP_Scripting::repl_start(void) { + if (terminal.session) { // it's already running, this is fine + return true; + } + + // nuke the old folder and all contents + struct stat st; + if ((AP::FS().stat(REPL_DIRECTORY, &st) == -1) && + (AP::FS().unlink(REPL_DIRECTORY) == -1) && + (errno != EEXIST)) { + gcs().send_text(MAV_SEVERITY_INFO, "Unable to delete old REPL %s", strerror(errno)); + } + + // create a new folder + AP::FS().mkdir(REPL_DIRECTORY); + // delete old files in case we couldn't + AP::FS().unlink(REPL_DIRECTORY "/in"); + AP::FS().unlink(REPL_DIRECTORY "/out"); + + // make the output pointer + terminal.output_fd = AP::FS().open(REPL_OUT, O_WRONLY|O_CREAT|O_TRUNC); + if (terminal.output_fd == -1) { + gcs().send_text(MAV_SEVERITY_INFO, "Unable to make new REPL"); + return false; + } + + terminal.session = true; + return true; +} + +void AP_Scripting::repl_stop(void) { + terminal.session = false; + // can't do any more cleanup here, closing the open FD's is the REPL's responsibility +} + void AP_Scripting::thread(void) { - lua_scripts *lua = new lua_scripts(_script_vm_exec_count, _script_heap_size, _debug_level); + lua_scripts *lua = new lua_scripts(_script_vm_exec_count, _script_heap_size, _debug_level, terminal); if (lua == nullptr || !lua->heap_allocated()) { gcs().send_text(MAV_SEVERITY_CRITICAL, "Unable to allocate scripting memory"); _init_failed = true; diff --git a/libraries/AP_Scripting/AP_Scripting.h b/libraries/AP_Scripting/AP_Scripting.h index d418d865b8e0e..19e63c044ba18 100644 --- a/libraries/AP_Scripting/AP_Scripting.h +++ b/libraries/AP_Scripting/AP_Scripting.h @@ -18,6 +18,8 @@ #include #include +#include +#include class AP_Scripting { @@ -37,7 +39,19 @@ class AP_Scripting static const struct AP_Param::GroupInfo var_info[]; + MAV_RESULT handle_command_int_packet(const mavlink_command_int_t &packet); + + struct terminal_s { + int output_fd; + off_t input_offset; + bool session; + } terminal; + private: + + bool repl_start(void); + void repl_stop(void); + void load_script(const char *filename); // load a script from a file void thread(void); // main script execution thread diff --git a/libraries/AP_Scripting/lua_repl.cpp b/libraries/AP_Scripting/lua_repl.cpp new file mode 100644 index 0000000000000..567365734d217 --- /dev/null +++ b/libraries/AP_Scripting/lua_repl.cpp @@ -0,0 +1,261 @@ +// this implements a Lua REPL, and is based off of a cut down version of +// lua/src/lua.c. It overall modified the functions to the minimum amount +// required, with the exception of fixing whitespace/indentation on if's + + +#include "lua_scripts.h" +#include "lua_generated_bindings.h" + +#include "lua/src/lua.h" +#include "lua/src/lauxlib.h" +#include "lua/src/lualib.h" + +#if !defined(LUA_MAXINPUT) +#define LUA_MAXINPUT 256 +#endif + +#if !defined(LUA_PROMPT) +#define LUA_PROMPT "> " +#define LUA_PROMPT2 ">> " +#endif + +extern const AP_HAL::HAL& hal; + +/* +** Message handler used to run all chunks +*/ +static int msghandler(lua_State *L) { + const char *msg = lua_tostring(L, 1); + if (msg == NULL) { /* is error object not a string? */ + if (luaL_callmeta(L, 1, "__tostring") && /* does it have a metamethod */ + lua_type(L, -1) == LUA_TSTRING) { /* that produces a string? */ + return 1; /* that is the message */ + } else { + msg = lua_pushfstring(L, "(error object is a %s value)", + luaL_typename(L, 1)); + } + } + luaL_traceback(L, L, msg, 1); /* append a standard traceback */ + return 1; /* return the traceback */ +} + + +/* +** Interface to 'lua_pcall', which sets appropriate message function +** and C-signal handler. Used to run all chunks. +*/ +int lua_scripts::docall(lua_State *L, int narg, int nres) { + int status; + int base = lua_gettop(L) - narg; /* function index */ + lua_rawgeti(L, LUA_REGISTRYINDEX, sandbox_ref); + lua_setupvalue(L, -2, 1); + lua_pushcfunction(L, msghandler); /* push message handler */ + lua_insert(L, base); /* put it under function and args */ + status = lua_pcall(L, narg, nres, base); + lua_remove(L, base); /* remove message handler from the stack */ + return status; +} + + +/* +** Returns the string to be used as a prompt by the interpreter. +*/ +const char * lua_scripts::get_prompt(lua_State *L, int firstline) { + const char *p; + lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2"); + p = lua_tostring(L, -1); + if (p == NULL) { + p = (firstline ? LUA_PROMPT : LUA_PROMPT2); + } + return p; +} + +/* mark in error messages for incomplete statements */ +#define EOFMARK "" +#define marklen (sizeof(EOFMARK)/sizeof(char) - 1) + + +/* +** Check whether 'status' signals a syntax error and the error +** message at the top of the stack ends with the above mark for +** incomplete statements. +*/ +int lua_scripts::incomplete(lua_State *L, int status) { + if (status == LUA_ERRSYNTAX) { + size_t lmsg; + const char *msg = lua_tolstring(L, -1, &lmsg); + if (lmsg >= marklen && strcmp(msg + lmsg - marklen, EOFMARK) == 0) { + lua_pop(L, 1); + return 1; + } + } + return 0; /* else... */ +} + + +/* +** Prompt the user, read a line, and push it into the Lua stack. +*/ +int lua_scripts::pushline(lua_State *L, int firstline) { + char buffer[LUA_MAXINPUT + 1] = {}; + ssize_t read_bytes; + size_t l = 0; + + // send prompt to the user + terminal_print(get_prompt(L, firstline)); + + while (terminal.session) { + // reseek to where we need input from, as invalid reads could have done weird stuff, and we want to start from the last valid input + int input_fd = AP::FS().open(REPL_IN, O_RDONLY); + if (input_fd != -1) { + AP::FS().lseek(input_fd, terminal.input_offset, SEEK_SET); + read_bytes = AP::FS().read(input_fd, buffer, ARRAY_SIZE(buffer) - 1); + AP::FS().close(input_fd); + if (read_bytes > 0) { + // locate the first newline + char * newline_chr = strchr(buffer, '\n'); + if (newline_chr == NULL) { + // we don't have something that looks like a newline, just keep reading till it's longer + read_bytes = 0; + } else { + newline_chr[0] = '\0'; + // only advance to the newline + l = strlen(buffer); + terminal.input_offset += l + 1; + break; + } + } + } + // wait for any input + hal.scheduler->delay(100); + } + + lua_pop(L, 1); /* remove prompt */ + lua_pushlstring(L, buffer, l); + return 1; +} + + +/* +** Try to compile line on the stack as 'return ;'; on return, stack +** has either compiled chunk or original line (if compilation failed). +*/ +int lua_scripts::addreturn(lua_State *L) { + const char *line = lua_tostring(L, -1); /* original line */ + const char *retline = lua_pushfstring(L, "return %s;", line); + int status = luaL_loadbuffer(L, retline, strlen(retline), "=stdin"); + if (status == LUA_OK) { + lua_remove(L, -2); /* remove modified line */ + } else { + lua_pop(L, 2); /* pop result from 'luaL_loadbuffer' and modified line */ + } + return status; +} + + +/* +** Read multiple lines until a complete Lua statement +*/ +int lua_scripts::multiline (lua_State *L) { + for (;;) { /* repeat until gets a complete statement */ + size_t len; + const char *line = lua_tolstring(L, 1, &len); /* get what it has */ + int status = luaL_loadbuffer(L, line, len, "=stdin"); /* try it */ + if (!incomplete(L, status) || !pushline(L, 0)) { + return status; /* cannot or should not try to add continuation line */ + } + lua_pushliteral(L, "\n"); /* add newline... */ + lua_insert(L, -2); /* ...between the two lines */ + lua_concat(L, 3); /* join them */ + } +} + + +/* +** Read a line and try to load (compile) it first as an expression (by +** adding "return " in front of it) and second as a statement. Return +** the final status of load/call with the resulting function (if any) +** in the top of the stack. +*/ +int lua_scripts::loadline(lua_State *L) { + int status; + lua_settop(L, 0); + if (!pushline(L, 1)) { + return -1; /* no input */ + } + if ((status = addreturn(L)) != LUA_OK) { /* 'return ...' did not work? */ + status = multiline(L); /* try as command, maybe with continuation lines */ + } else { + } + lua_remove(L, 1); /* remove line from the stack */ + lua_assert(lua_gettop(L) == 1); + return status; +} + +// push the tring into the terminal, blocks until it's queued +void lua_scripts::terminal_print(const char *str) { + if ((AP::FS().write(terminal.output_fd, str, strlen(str)) == -1) || + (AP::FS().fsync(terminal.output_fd) != 0)) { + terminal.session = false; + } +} + +/* +** Prints (calling the Lua 'print' function) any values on the stack +*/ +void lua_scripts::l_print(lua_State *L) { + int n = lua_gettop(L); + if (n > 0) { /* any result to be printed? */ + luaL_checkstack(L, LUA_MINSTACK, "too many results to print"); + // grab all the internal functions via the sandbox + lua_rawgeti(L, LUA_REGISTRYINDEX, sandbox_ref); + lua_getfield(L, -1, "string"); + lua_getfield(L, -1, "format"); + lua_insert(L, 1); + lua_remove(L, -2); + lua_getfield(L, -1, "rep"); + lua_remove(L, -2); + lua_pushliteral(L, "%s"); + lua_pushinteger(L, n); + lua_pushliteral(L, "\t"); + if (lua_pcall(L, 3, 1, 0) != LUA_OK) { + // should never happen + lua_error(L); + } + lua_insert(L, 2); + if (lua_pcall(L, n + 1, 1, 0) != LUA_OK) { + terminal_print(lua_pushfstring(L, "error calling 'print' (%s)\n", lua_tostring(L, -1))); + } else { + terminal_print(lua_pushfstring(L, "%s\n", lua_tostring(L, -1))); + } + } +} + +/* +** Do the REPL: repeatedly read (load) a line, evaluate (call) it, and +** print any results. +*/ +void lua_scripts::doREPL(lua_State *L) { + int status; + // clear out any old script results + reset_loop_overtime(L); + // prep the sandbox + create_sandbox(L); + sandbox_ref = luaL_ref(L, LUA_REGISTRYINDEX); + terminal.input_offset = 0; + while (((status = loadline(L)) != -1) && terminal.session) { + if (status == LUA_OK) { + status = docall(L, 0, LUA_MULTRET); + } + if (status == LUA_OK) { + l_print(L); + } else { + terminal_print(lua_pushfstring(L, "%s\n", lua_tostring(L, -1))); + } + reset_loop_overtime(L); + } + lua_settop(L, 0); /* clear stack */ + luaL_unref(L, LUA_REGISTRYINDEX, sandbox_ref); + repl_cleanup(); +} + diff --git a/libraries/AP_Scripting/lua_scripts.cpp b/libraries/AP_Scripting/lua_scripts.cpp index c465528181aaf..9188ec62b24a9 100644 --- a/libraries/AP_Scripting/lua_scripts.cpp +++ b/libraries/AP_Scripting/lua_scripts.cpp @@ -33,9 +33,10 @@ extern const AP_HAL::HAL& hal; bool lua_scripts::overtime; jmp_buf lua_scripts::panic_jmp; -lua_scripts::lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level) +lua_scripts::lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level, struct AP_Scripting::terminal_s &_terminal) : _vm_steps(vm_steps), - _debug_level(debug_level) { + _debug_level(debug_level), + terminal(_terminal) { _heap = hal.util->allocate_heap_memory(heap_size); } @@ -52,6 +53,7 @@ void lua_scripts::hook(lua_State *L, lua_Debug *ar) { int lua_scripts::atpanic(lua_State *L) { gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: Panic: %s", lua_tostring(L, -1)); hal.console->printf("Lua: Panic: %s\n", lua_tostring(L, -1)); + printf("Lua: Panic: %s\n", lua_tostring(L, -1)); longjmp(panic_jmp, 1); return 0; } @@ -91,8 +93,16 @@ lua_scripts::script_info *lua_scripts::load_script(lua_State *L, char *filename) new_script->name = filename; new_script->next = nullptr; + create_sandbox(L); + lua_setupvalue(L, -2, 1); + + new_script->lua_ref = luaL_ref(L, LUA_REGISTRYINDEX); // cache the reference + new_script->next_run_ms = AP_HAL::millis64() - 1; // force the script to be stale - // find and create a sandbox for the new chunk + return new_script; +} + +void lua_scripts::create_sandbox(lua_State *L) { lua_newtable(L); luaopen_base_sandbox(L); lua_pushstring(L, "math"); @@ -112,12 +122,7 @@ lua_scripts::script_info *lua_scripts::load_script(lua_State *L, char *filename) lua_settable(L, -3); load_lua_bindings(L); load_generated_sandbox(L); - lua_setupvalue(L, -2, 1); - new_script->lua_ref = luaL_ref(L, LUA_REGISTRYINDEX); // cache the reference - new_script->next_run_ms = AP_HAL::millis64() - 1; // force the script to be stale - - return new_script; } void lua_scripts::load_all_scripts_in_dir(lua_State *L, const char *dirname) { @@ -164,6 +169,13 @@ void lua_scripts::load_all_scripts_in_dir(lua_State *L, const char *dirname) { AP::FS().closedir(d); } +void lua_scripts::reset_loop_overtime(lua_State *L) { + overtime = false; + // reset the hook to clear the counter + const int32_t vm_steps = MAX(_vm_steps, 1000); + lua_sethook(L, hook, LUA_MASKCOUNT, vm_steps); +} + void lua_scripts::run_next_script(lua_State *L) { if (scripts == nullptr) { #if defined(AP_SCRIPTING_CHECKS) && AP_SCRIPTING_CHECKS >= 1 @@ -172,16 +184,12 @@ void lua_scripts::run_next_script(lua_State *L) { return; } - // reset the current script tracking information - overtime = false; - // strip the selected script out of the list script_info *script = scripts; scripts = script->next; // reset the hook to clear the counter - const int32_t vm_steps = MAX(_vm_steps, 1000); - lua_sethook(L, hook, LUA_MASKCOUNT, vm_steps); + reset_loop_overtime(L); // store top of stack so we can calculate the number of return values int stack_top = lua_gettop(L); @@ -192,7 +200,7 @@ void lua_scripts::run_next_script(lua_State *L) { if(lua_pcall(L, 0, LUA_MULTRET, 0)) { if (overtime) { // script has consumed an excessive amount of CPU time - gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: %s exceeded time limit (%d)", script->name, (int)vm_steps); + gcs().send_text(MAV_SEVERITY_CRITICAL, "Lua: %s exceeded time limit", script->name); remove_script(L, script); } else { hal.console->printf("Lua: Error: %s\n", lua_tostring(L, -1)); @@ -314,6 +322,19 @@ void *lua_scripts::alloc(void *ud, void *ptr, size_t osize, size_t nsize) { return hal.util->heap_realloc(_heap, ptr, nsize); } +void lua_scripts::repl_cleanup (void) { + if (terminal.session) { + terminal.session = false; + if (terminal.output_fd != -1) { + AP::FS().close(terminal.output_fd); + terminal.output_fd = -1; + AP::FS().unlink(REPL_DIRECTORY "/in"); + AP::FS().unlink(REPL_DIRECTORY "/out"); + AP::FS().unlink(REPL_DIRECTORY); + } + } +} + void lua_scripts::run(void) { bool succeeded_initial_load = false; @@ -336,6 +357,8 @@ void lua_scripts::run(void) { } scripts = nullptr; overtime = false; + // end any open REPL sessions + repl_cleanup(); } lua_state = lua_newstate(alloc, NULL); @@ -355,6 +378,12 @@ void lua_scripts::run(void) { succeeded_initial_load = true; while (AP_Scripting::get_singleton()->enabled()) { + // handle terminal data if we have any + if (terminal.session) { + doREPL(L); + continue; + } + #if defined(AP_SCRIPTING_CHECKS) && AP_SCRIPTING_CHECKS >= 1 if (lua_gettop(L) != 0) { AP_HAL::panic("Lua: Stack should be empty before running scripts"); diff --git a/libraries/AP_Scripting/lua_scripts.h b/libraries/AP_Scripting/lua_scripts.h index 361fa09f3292d..f2f7be870877e 100644 --- a/libraries/AP_Scripting/lua_scripts.h +++ b/libraries/AP_Scripting/lua_scripts.h @@ -20,11 +20,28 @@ #include #include "lua_bindings.h" +#include + +#ifndef REPL_DIRECTORY + #if HAL_OS_FATFS_IO + #define REPL_DIRECTORY "/APM/repl" + #else + #define REPL_DIRECTORY "./repl" + #endif //HAL_OS_FATFS_IO +#endif // REPL_DIRECTORY + +#ifndef REPL_IN + #define REPL_IN REPL_DIRECTORY "/in" +#endif // REPL_IN + +#ifndef REPL_OUT + #define REPL_OUT REPL_DIRECTORY "/out" +#endif // REPL_OUT class lua_scripts { public: - lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level); + lua_scripts(const AP_Int32 &vm_steps, const AP_Int32 &heap_size, const AP_Int8 &debug_level, struct AP_Scripting::terminal_s &_terminal); /* Do not allow copies */ lua_scripts(const lua_scripts &other) = delete; @@ -39,6 +56,10 @@ class lua_scripts static bool overtime; // script exceeded it's execution slot, and we are bailing out private: + void create_sandbox(lua_State *L); + + void repl_cleanup(void); + typedef struct script_info { int lua_ref; // reference to the loaded script object uint64_t next_run_ms; // time (in milliseconds) the script should next be run at @@ -48,6 +69,8 @@ class lua_scripts script_info *load_script(lua_State *L, char *filename); + void reset_loop_overtime(lua_State *L); + void load_all_scripts_in_dir(lua_State *L, const char *dirname); void run_next_script(lua_State *L); @@ -57,6 +80,20 @@ class lua_scripts // reschedule the script for execution. It is assumed the script is not in the list already void reschedule_script(script_info *script); + // REPL stuff + struct AP_Scripting::terminal_s &terminal; + void doREPL(lua_State *L); + void l_print(lua_State *L); + void terminal_print(const char *str); + int loadline(lua_State *L); + int multiline(lua_State *L); + int addreturn(lua_State *L); + int pushline(lua_State *L, int firstline); + int incomplete(lua_State *L, int status); + const char * get_prompt(lua_State *L, int firstline); + int docall(lua_State *L, int narg, int nres); + int sandbox_ref; + script_info *scripts; // linked list of scripts to be run, sorted by next run time (soonest first) // hook will be run when CPU time for a script is exceeded From 71efe3ded1fc404032adf52d889a776bab4eedef Mon Sep 17 00:00:00 2001 From: Michael du Breuil Date: Mon, 17 Feb 2020 19:11:46 -0700 Subject: [PATCH 2/3] modules: Update MAVLink version --- modules/mavlink | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mavlink b/modules/mavlink index fa5ce823efc5d..45b61f5945387 160000 --- a/modules/mavlink +++ b/modules/mavlink @@ -1 +1 @@ -Subproject commit fa5ce823efc5d2786b26e488311d0619e43c1e75 +Subproject commit 45b61f594538715e2c85c548dfaa48daa6c4026c From 8e9deafde9d99fecd9633cddaf3c122b731c78cd Mon Sep 17 00:00:00 2001 From: Michael du Breuil Date: Mon, 10 Feb 2020 21:25:15 -0700 Subject: [PATCH 3/3] GCS_MAVLink: Allow scripting to start/stop the REPL --- libraries/GCS_MAVLink/GCS_Common.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libraries/GCS_MAVLink/GCS_Common.cpp b/libraries/GCS_MAVLink/GCS_Common.cpp index 30914585c785b..ef12122958aa5 100644 --- a/libraries/GCS_MAVLink/GCS_Common.cpp +++ b/libraries/GCS_MAVLink/GCS_Common.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -4053,6 +4054,16 @@ MAV_RESULT GCS_MAVLINK::handle_command_int_packet(const mavlink_command_int_t &p return handle_command_do_set_roi_sysid(packet); case MAV_CMD_DO_SET_HOME: return handle_command_int_do_set_home(packet); +#ifdef ENABLE_SCRIPTING + case MAV_CMD_SCRIPTING: + { + AP_Scripting *scripting = AP_Scripting::get_singleton(); + if (scripting == nullptr) { + return MAV_RESULT_UNSUPPORTED; + } + return scripting->handle_command_int_packet(packet); + } +#endif // ENABLE_SCRIPTING default: break; }