diff --git a/.circleci/config.yml b/.circleci/config.yml index e2382dfd7..53beb19fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,7 @@ commands: - run: name: Setup build system command: | + clang-format --version ./opt/system-setup.py checkout-all: @@ -220,7 +221,7 @@ commands: jobs: lint: docker: - - image: redislabsmodules/llvm-toolset:latest + - image: redisfab/rmbuilder:6.2.5-x64-buster steps: - abort_for_docs - abort_for_noci diff --git a/opt/build/docker/dockerfile-gpu-test.tmpl b/opt/build/docker/dockerfile-gpu-test.tmpl index 2ac7f01a7..c25490162 100644 --- a/opt/build/docker/dockerfile-gpu-test.tmpl +++ b/opt/build/docker/dockerfile-gpu-test.tmpl @@ -36,6 +36,8 @@ COPY ./tests/flow/tests_setup/test_requirements.txt tests/flow/tests_setup/ COPY ./tests/flow/tests_setup/Install_RedisGears.sh tests/flow/tests_setup/ COPY ./get_deps.sh . +RUN apt-get -q install -y git + RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3 RUN set -e ;\ diff --git a/opt/build/docker/dockerfile.tmpl b/opt/build/docker/dockerfile.tmpl index 646aebfac..c821e970a 100755 --- a/opt/build/docker/dockerfile.tmpl +++ b/opt/build/docker/dockerfile.tmpl @@ -30,8 +30,10 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ ADD ./tests/flow/ tests/flow/ -RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/readies/bin/getupdates +RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y git; fi +RUN if [ ! -z $(command -v yum) ]; then yum install -y git; fi +RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/opt/clang-check-all.sh b/opt/clang-check-all.sh index 45c601317..178da77ad 100755 --- a/opt/clang-check-all.sh +++ b/opt/clang-check-all.sh @@ -3,8 +3,8 @@ # exit immediatly on error ( no need to keep checking ) set -e -CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) -CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) +CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) +CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) for filename in $CLANG_FMT_SRCS; do echo "Checking $filename" diff --git a/opt/clang-format-all.sh b/opt/clang-format-all.sh index 93ce7bb96..3412c62c7 100755 --- a/opt/clang-format-all.sh +++ b/opt/clang-format-all.sh @@ -3,8 +3,8 @@ # exit immediatly on error ( no need to keep checking ) set -e -CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) -CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc'-o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) +CLANG_FMT_SRCS=$(find ../src/ \( -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) +CLANG_FMT_TESTS=$(find ../tests/ \( -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o -name '*.h' -o -name '*.hh' -o -name '*.hpp' \)) for filename in $CLANG_FMT_SRCS; do clang-format --verbose -style=file -i $filename diff --git a/opt/readies b/opt/readies index 89be26742..5f12f2619 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 89be267427c7dfcfaab4064942ef0f595f6b1fa3 +Subproject commit 5f12f261993e756f7c1757fdfb7afdd60788d5d1 diff --git a/src/backends/libtflite_c/tflite_c.cpp b/src/backends/libtflite_c/tflite_c.cpp index 2ad239fea..3bffd7d92 100644 --- a/src/backends/libtflite_c/tflite_c.cpp +++ b/src/backends/libtflite_c/tflite_c.cpp @@ -53,7 +53,6 @@ static DLDataType getDLDataType(const TfLiteTensor *tensor) { return dtype; } - static DLDevice getDLDevice(const TfLiteTensor *tensor, const int64_t &device_id) { DLDevice device; device.device_id = device_id; @@ -61,7 +60,6 @@ static DLDevice getDLDevice(const TfLiteTensor *tensor, const int64_t &device_id return device; } - size_t dltensorBytes(DLManagedTensor *t) { int64_t *shape = t->dl_tensor.shape; size_t len = 1; @@ -80,15 +78,15 @@ void copyToTfLiteTensor(std::shared_ptr interpreter, int tf size_t nbytes = dltensorBytes(input); DLDataType dltensor_type = input->dl_tensor.dtype; const char *type_mismatch_msg = "Input tensor type doesn't match the type expected" - " by the model definition"; + " by the model definition"; switch (tensor->type) { case kTfLiteUInt8: - if (dltensor_type.code != kDLUInt || dltensor_type.bits != 8) { - throw std::logic_error(type_mismatch_msg); - } - memcpy(interpreter->typed_tensor(tflite_input), input->dl_tensor.data, nbytes); - break; + if (dltensor_type.code != kDLUInt || dltensor_type.bits != 8) { + throw std::logic_error(type_mismatch_msg); + } + memcpy(interpreter->typed_tensor(tflite_input), input->dl_tensor.data, nbytes); + break; case kTfLiteInt64: if (dltensor_type.code != kDLInt || dltensor_type.bits != 64) { throw std::logic_error(type_mismatch_msg); @@ -273,51 +271,47 @@ extern "C" void *tfliteLoadModel(const char *graph, size_t graphlen, DLDeviceTyp return ctx; } -extern "C" size_t tfliteModelNumInputs(void* ctx, char** error) { - ModelContext *ctx_ = (ModelContext*) ctx; +extern "C" size_t tfliteModelNumInputs(void *ctx, char **error) { + ModelContext *ctx_ = (ModelContext *)ctx; size_t ret = 0; try { auto interpreter = ctx_->interpreter; - ret = interpreter->inputs().size(); - } - catch(std::exception ex) { + ret = interpreter->inputs().size(); + } catch (std::exception ex) { _setError(ex.what(), error); } return ret; } -extern "C" const char* tfliteModelInputNameAtIndex(void* modelCtx, size_t index, char** error) { - ModelContext *ctx_ = (ModelContext*) modelCtx; - const char* ret = NULL; +extern "C" const char *tfliteModelInputNameAtIndex(void *modelCtx, size_t index, char **error) { + ModelContext *ctx_ = (ModelContext *)modelCtx; + const char *ret = NULL; try { ret = ctx_->interpreter->GetInputName(index); - } - catch(std::exception ex) { + } catch (std::exception ex) { _setError(ex.what(), error); } return ret; } -extern "C" size_t tfliteModelNumOutputs(void* ctx, char** error) { - ModelContext *ctx_ = (ModelContext*) ctx; +extern "C" size_t tfliteModelNumOutputs(void *ctx, char **error) { + ModelContext *ctx_ = (ModelContext *)ctx; size_t ret = 0; try { auto interpreter = ctx_->interpreter; - ret = interpreter->outputs().size(); - } - catch(std::exception ex) { + ret = interpreter->outputs().size(); + } catch (std::exception ex) { _setError(ex.what(), error); } return ret; } -extern "C" const char* tfliteModelOutputNameAtIndex(void* modelCtx, size_t index, char** error) { - ModelContext *ctx_ = (ModelContext*) modelCtx; - const char* ret = NULL; +extern "C" const char *tfliteModelOutputNameAtIndex(void *modelCtx, size_t index, char **error) { + ModelContext *ctx_ = (ModelContext *)modelCtx; + const char *ret = NULL; try { ret = ctx_->interpreter->GetOutputName(index); - } - catch(std::exception ex) { + } catch (std::exception ex) { _setError(ex.what(), error); } return ret; @@ -352,11 +346,11 @@ extern "C" void tfliteRunModel(void *ctx, long n_inputs, DLManagedTensor **input bool need_reallocation = false; std::vector dims; for (size_t i = 0; i < tflite_inputs.size(); i++) { - const TfLiteTensor* tflite_tensor = interpreter->tensor(tflite_inputs[i]); + const TfLiteTensor *tflite_tensor = interpreter->tensor(tflite_inputs[i]); int64_t ndim = inputs[i]->dl_tensor.ndim; int64_t *shape = inputs[i]->dl_tensor.shape; dims.resize(ndim); - for (size_t j=0; j < ndim; j++) { + for (size_t j = 0; j < ndim; j++) { dims[j] = shape[j]; } if (!tflite::EqualArrayAndTfLiteIntArray(tflite_tensor->dims, dims.size(), dims.data())) { diff --git a/src/backends/libtorch_c/torch_c.cpp b/src/backends/libtorch_c/torch_c.cpp index cbf8f2906..abea839fd 100644 --- a/src/backends/libtorch_c/torch_c.cpp +++ b/src/backends/libtorch_c/torch_c.cpp @@ -165,9 +165,7 @@ torch::Tensor fromDLPack(const DLTensor *src) { torch::device(device).dtype(stype)); } -extern "C" void torchRegisterRedisOps(void) { - registerRedisOps(); -} +extern "C" void torchRegisterRedisOps(void) { registerRedisOps(); } extern "C" void torchTensorFromRAITensor(RAI_Tensor *src, void *torch_tensor) { DLTensor *dl_tensor = RedisAI_TensorGetDLTensor(src); @@ -177,18 +175,14 @@ extern "C" void torchTensorFromRAITensor(RAI_Tensor *src, void *torch_tensor) { // Capture the RAI_Tensor to be able to release it once torch is done with // the tensor that we are about to create (to avoid copying of the blob). - auto free_tensor = [src](void *data) { - RedisAI_TensorFree(src); - }; + auto free_tensor = [src](void *data) { RedisAI_TensorFree(src); }; // Create torch tensor with the tensor's blob, and send a deleter callback // for torch to use to release the RAI_Tensor when it finishes. - *static_cast(torch_tensor) = - torch::Tensor(torch::from_blob(dl_tensor->data, - at::IntArrayRef(dl_tensor->shape, dl_tensor->ndim), - at::IntArrayRef(dl_tensor->strides, dl_tensor->ndim), - free_tensor, - torch::device(device).dtype(stype))); + *static_cast(torch_tensor) = torch::Tensor( + torch::from_blob(dl_tensor->data, at::IntArrayRef(dl_tensor->shape, dl_tensor->ndim), + at::IntArrayRef(dl_tensor->strides, dl_tensor->ndim), free_tensor, + torch::device(device).dtype(stype))); } struct ATenDLMTensor { @@ -198,7 +192,7 @@ struct ATenDLMTensor { void deleter(DLManagedTensor *arg) { delete static_cast(arg->manager_ctx); - RedisModule_Free(arg); + RedisModule_Free(arg); } DLManagedTensor *toManagedDLPack(const torch::Tensor &src_) { @@ -208,7 +202,7 @@ DLManagedTensor *toManagedDLPack(const torch::Tensor &src_) { atDLMTensor->tensor.manager_ctx = atDLMTensor; atDLMTensor->tensor.deleter = &deleter; atDLMTensor->tensor.dl_tensor.data = src.data_ptr(); - int64_t device_id = -1; // This should be used for the default 'CPU' device. + int64_t device_id = -1; // This should be used for the default 'CPU' device. if (src.is_cuda()) { device_id = src.get_device(); } @@ -233,11 +227,13 @@ struct ModuleContext { int64_t device_id; }; -static void torchHandlOutputs(torch::jit::Stack& stack, const char* fnName, long nOutputs, DLManagedTensor **outputs) { +static void torchHandlOutputs(torch::jit::Stack &stack, const char *fnName, long nOutputs, + DLManagedTensor **outputs) { torch::DeviceType output_device_type = torch::kCPU; torch::Device output_device(output_device_type, -1); - if(nOutputs == 0) return; + if (nOutputs == 0) + return; int count = 0; for (size_t i = 0; i < stack.size(); i++) { if (count > nOutputs - 1) { @@ -274,7 +270,8 @@ static void torchHandlOutputs(torch::jit::Stack& stack, const char* fnName, long } } -void torchRunModule(ModuleContext *ctx, const char *fnName, torch::jit::Stack& stack, long nOutputs, DLManagedTensor **outputs){ +void torchRunModule(ModuleContext *ctx, const char *fnName, torch::jit::Stack &stack, long nOutputs, + DLManagedTensor **outputs) { if (ctx->module) { torch::NoGradGuard guard; @@ -291,34 +288,28 @@ void torchRunModule(ModuleContext *ctx, const char *fnName, torch::jit::Stack& s } // namespace -extern "C" void* torchCompileScript(const char* script, DLDeviceType device, int64_t device_id, - char **error) -{ - ModuleContext* ctx = new ModuleContext(); - ctx->device = device; - ctx->device_id = device_id; - try { - auto cu = std::make_shared(); - cu->define( - c10::nullopt, - script, - torch::jit::script::redisResolver(), - nullptr); - auto aten_device_type = getATenDeviceType(device); - - if (aten_device_type == at::DeviceType::CUDA && !torch::cuda::is_available()) { - throw std::logic_error("GPU requested but Torch couldn't find CUDA"); +extern "C" void *torchCompileScript(const char *script, DLDeviceType device, int64_t device_id, + char **error) { + ModuleContext *ctx = new ModuleContext(); + ctx->device = device; + ctx->device_id = device_id; + try { + auto cu = std::make_shared(); + cu->define(c10::nullopt, script, torch::jit::script::redisResolver(), nullptr); + auto aten_device_type = getATenDeviceType(device); + + if (aten_device_type == at::DeviceType::CUDA && !torch::cuda::is_available()) { + throw std::logic_error("GPU requested but Torch couldn't find CUDA"); + } + ctx->cu = cu; + ctx->module = nullptr; + + } catch (std::exception &e) { + *error = RedisModule_Strdup(e.what()); + delete ctx; + return NULL; } - ctx->cu = cu; - ctx->module = nullptr; - - } - catch(std::exception& e) { - *error = RedisModule_Strdup(e.what()); - delete ctx; - return NULL; - } - return ctx; + return ctx; } extern "C" void *torchLoadModel(const char *graph, size_t graphlen, DLDeviceType device, @@ -340,7 +331,7 @@ extern "C" void *torchLoadModel(const char *graph, size_t graphlen, DLDeviceType ctx->module = module; ctx->cu = nullptr; } catch (std::exception &e) { - *error = RedisModule_Strdup(e.what()); + *error = RedisModule_Strdup(e.what()); delete ctx; return NULL; } @@ -349,52 +340,55 @@ extern "C" void *torchLoadModel(const char *graph, size_t graphlen, DLDeviceType static torch::DeviceType getDeviceType(ModuleContext *ctx) { switch (ctx->device) { - case kDLCPU: - return torch::kCPU; - case kDLCUDA: - return torch::kCUDA; - default: - throw std::runtime_error(std::string("Unsupported device ") + std::to_string(ctx->device)); + case kDLCPU: + return torch::kCPU; + case kDLCUDA: + return torch::kCUDA; + default: + throw std::runtime_error(std::string("Unsupported device ") + std::to_string(ctx->device)); } } - extern "C" void torchRunScript(void *scriptCtx, const char *fnName, - TorchFunctionInputCtx* inputsCtx, - DLManagedTensor **outputs,long nOutputs, - char **error) { + TorchFunctionInputCtx *inputsCtx, DLManagedTensor **outputs, + long nOutputs, char **error) { ModuleContext *ctx = (ModuleContext *)scriptCtx; try { torch::DeviceType device_type = getDeviceType(ctx); torch::Device device(device_type, ctx->device_id); torch::jit::Stack stack; - if(!inputsCtx->hasEntryPoint) { - /* In case of no entry point, this might be due to a usage in a script set by the deprecated API. - * In this case, until SCRIPTSET is EOL we will allow functions, called by SCRIPTRUN or SCRIPTEXECUTE, and those - * functions are not in the endpoint set to be executed in "best effort" manner. + if (!inputsCtx->hasEntryPoint) { + /* In case of no entry point, this might be due to a usage in a script set by the + * deprecated API. In this case, until SCRIPTSET is EOL we will allow functions, called + * by SCRIPTRUN or SCRIPTEXECUTE, and those functions are not in the endpoint set to be + * executed in "best effort" manner. */ - if(!torchScript_FunctionExists(ctx, fnName)) { + if (!torchScript_FunctionExists(ctx, fnName)) { throw std::runtime_error(std::string("Function does not exist: ") + fnName); } size_t nArgs = torchScript_FunctionArgumentCountByFunctionName(ctx, fnName); - if(nArgs > inputsCtx->tensorCount) { - throw std::runtime_error(std::string("Wrong number of inputs provided to function: ") + fnName); + if (nArgs > inputsCtx->tensorCount) { + throw std::runtime_error( + std::string("Wrong number of inputs provided to function: ") + fnName); } - for (size_t i= 0; i < nArgs; i++) { - TorchScriptFunctionArgumentType argType = torchScript_FunctionArgumentTypeByFunctionName(ctx, fnName, i); + for (size_t i = 0; i < nArgs; i++) { + TorchScriptFunctionArgumentType argType = + torchScript_FunctionArgumentTypeByFunctionName(ctx, fnName, i); // Argument can be either a tensor or a tensor list. - if((argType!= TENSOR && argType != TENSOR_LIST)) { - throw std::runtime_error(std::string("Unsupported input type in function definition: ") + fnName); + if ((argType != TENSOR && argType != TENSOR_LIST)) { + throw std::runtime_error( + std::string("Unsupported input type in function definition: ") + fnName); } - if(argType == TENSOR) { + if (argType == TENSOR) { // In case of tensor DLTensor *input = &(inputsCtx->tensorInputs[i]->dl_tensor); torch::Tensor tensor = fromDLPack(input); tensor.to(device); stack.push_back(tensor); } else { - // In case of a tensor list, this is the last argument, add all tensors and break. + // In case of a tensor list, this is the last argument, add all tensors and + // break. std::vector tensors; for (size_t j = i; j < inputsCtx->tensorCount; j++) { DLTensor *input = &(inputsCtx->tensorInputs[j]->dl_tensor); @@ -406,37 +400,35 @@ extern "C" void torchRunScript(void *scriptCtx, const char *fnName, break; } } - } - else { - std::vector tensors; - for (size_t i = 0; i < inputsCtx->tensorCount; i++) { - DLTensor *input = &(inputsCtx->tensorInputs[i]->dl_tensor); - torch::Tensor tensor = fromDLPack(input); - tensor.to(device); - tensors.emplace_back(tensor); - } - stack.push_back(tensors); - - std::vector keys; - for(size_t i=0; i < inputsCtx->keysCount; i++) { - size_t len; - const char* cstr = RedisModule_StringPtrLen(inputsCtx->keys[i], &len); - torch::string str = torch::string(cstr); - keys.emplace_back(str); - } - stack.push_back(keys); - - std::vector args; - for(size_t i=0; i < inputsCtx->argsCount; i++) { - size_t len; - const char* cstr = RedisModule_StringPtrLen(inputsCtx->args[i], &len); - torch::string str = torch::string(cstr); - args.emplace_back(str); - } - stack.push_back(args); + } else { + std::vector tensors; + for (size_t i = 0; i < inputsCtx->tensorCount; i++) { + DLTensor *input = &(inputsCtx->tensorInputs[i]->dl_tensor); + torch::Tensor tensor = fromDLPack(input); + tensor.to(device); + tensors.emplace_back(tensor); + } + stack.push_back(tensors); + + std::vector keys; + for (size_t i = 0; i < inputsCtx->keysCount; i++) { + size_t len; + const char *cstr = RedisModule_StringPtrLen(inputsCtx->keys[i], &len); + torch::string str = torch::string(cstr); + keys.emplace_back(str); + } + stack.push_back(keys); + + std::vector args; + for (size_t i = 0; i < inputsCtx->argsCount; i++) { + size_t len; + const char *cstr = RedisModule_StringPtrLen(inputsCtx->args[i], &len); + torch::string str = torch::string(cstr); + args.emplace_back(str); + } + stack.push_back(args); } - torchRunModule(ctx, fnName, stack, nOutputs, outputs); } catch (std::exception &e) { *error = RedisModule_Strdup(e.what()); @@ -510,30 +502,30 @@ extern "C" void torchSetIntraOpThreads(int num_threads, char **error) { } } -extern "C" size_t torchModelNumInputs(void *modelCtx, char** error) { +extern "C" size_t torchModelNumInputs(void *modelCtx, char **error) { ModuleContext *ctx = (ModuleContext *)modelCtx; size_t ninputs = 0; try { - const c10::FunctionSchema& schema = ctx->module->get_method("forward").function().getSchema(); + const c10::FunctionSchema &schema = + ctx->module->get_method("forward").function().getSchema(); // First argument is `self` - ninputs = schema.arguments().size() - 1; - } - catch(std::exception ex) { - int printed = asprintf(error, "Erorr while trying to retrive model inputs number: %s", ex.what()); + ninputs = schema.arguments().size() - 1; + } catch (std::exception ex) { + int printed = + asprintf(error, "Erorr while trying to retrive model inputs number: %s", ex.what()); } return ninputs; } -static int getArgumentTensorCount(const c10::Argument& arg){ - switch (arg.type()->kind()) - { +static int getArgumentTensorCount(const c10::Argument &arg) { + switch (arg.type()->kind()) { case c10::TypeKind::TensorType: return 1; break; case c10::TypeKind::TupleType: { int count = 0; - for(auto const& obj: arg.type()->containedTypes()) { - if(obj->kind() == c10::TypeKind::TensorType) { + for (auto const &obj : arg.type()->containedTypes()) { + if (obj->kind() == c10::TypeKind::TensorType) { count++; } } @@ -542,36 +534,35 @@ static int getArgumentTensorCount(const c10::Argument& arg){ case c10::TypeKind::ListType: { return arg.N().value(); } - + default: return 0; } } -static TorchScriptFunctionArgumentType getArgumentType(const c10::Argument& arg){ - switch (arg.type()->kind()) - { +static TorchScriptFunctionArgumentType getArgumentType(const c10::Argument &arg) { + switch (arg.type()->kind()) { case c10::TypeKind::TensorType: return TENSOR; - case c10::TypeKind::IntType: + case c10::TypeKind::IntType: return INT; case c10::TypeKind::FloatType: return FLOAT; - case c10::TypeKind::StringType: + case c10::TypeKind::StringType: return STRING; case c10::TypeKind::ListType: { c10::ListTypePtr lt = arg.type()->cast(); - switch(lt->getElementType()->kind()) { - case c10::TypeKind::TensorType: - return TENSOR_LIST; - case c10::TypeKind::IntType: - return INT_LIST; - case c10::TypeKind::FloatType: - return FLOAT_LIST; - case c10::TypeKind::StringType: - return STRING_LIST; - default: - return UNKOWN; + switch (lt->getElementType()->kind()) { + case c10::TypeKind::TensorType: + return TENSOR_LIST; + case c10::TypeKind::IntType: + return INT_LIST; + case c10::TypeKind::FloatType: + return FLOAT_LIST; + case c10::TypeKind::StringType: + return STRING_LIST; + default: + return UNKOWN; } } default: @@ -579,60 +570,64 @@ static TorchScriptFunctionArgumentType getArgumentType(const c10::Argument& arg } } -extern "C" size_t torchModelNumOutputs(void *modelCtx, char** error) { +extern "C" size_t torchModelNumOutputs(void *modelCtx, char **error) { ModuleContext *ctx = (ModuleContext *)modelCtx; size_t noutputs = 0; try { - const c10::FunctionSchema& schema = ctx->module->get_method("forward").function().getSchema(); - for (auto const& arg :schema.returns()){ - noutputs += getArgumentTensorCount(arg); + const c10::FunctionSchema &schema = + ctx->module->get_method("forward").function().getSchema(); + for (auto const &arg : schema.returns()) { + noutputs += getArgumentTensorCount(arg); } - } - catch(std::exception ex) { - int printed = asprintf(error, "Erorr while trying to retrive model outputs number: %s", ex.what()); + } catch (std::exception ex) { + int printed = + asprintf(error, "Erorr while trying to retrive model outputs number: %s", ex.what()); } return noutputs; } -extern "C" const char* torchModelInputNameAtIndex(void* modelCtx, size_t index, char** error) { +extern "C" const char *torchModelInputNameAtIndex(void *modelCtx, size_t index, char **error) { ModuleContext *ctx = (ModuleContext *)modelCtx; - const char* ret = NULL; + const char *ret = NULL; try { - const c10::FunctionSchema& schema = ctx->module->get_method("forward").function().getSchema(); - ret = schema.arguments()[index + 1].name().c_str(); - } - catch(std::exception ex) { - int printed = asprintf(error, "Erorr while trying to retrive model intput at index %ld: %s", index, ex.what()); + const c10::FunctionSchema &schema = + ctx->module->get_method("forward").function().getSchema(); + ret = schema.arguments()[index + 1].name().c_str(); + } catch (std::exception ex) { + int printed = asprintf(error, "Erorr while trying to retrive model intput at index %ld: %s", + index, ex.what()); } return ret; } -extern "C" size_t torchScript_FunctionCount(void* scriptCtx) { +extern "C" size_t torchScript_FunctionCount(void *scriptCtx) { ModuleContext *ctx = (ModuleContext *)scriptCtx; return ctx->cu->get_functions().size(); } -extern "C" const char* torchScript_FunctionName(void* scriptCtx, size_t fn_index) { +extern "C" const char *torchScript_FunctionName(void *scriptCtx, size_t fn_index) { ModuleContext *ctx = (ModuleContext *)scriptCtx; - std::vector functions = ctx->cu->get_functions(); + std::vector functions = ctx->cu->get_functions(); return functions[fn_index]->name().c_str(); } -extern "C" size_t torchScript_FunctionArgumentCountByFunctionName(void *scriptCtx, const char* functionName) { +extern "C" size_t torchScript_FunctionArgumentCountByFunctionName(void *scriptCtx, + const char *functionName) { ModuleContext *ctx = (ModuleContext *)scriptCtx; - torch::jit::Function* function = ctx->cu->find_function(functionName); + torch::jit::Function *function = ctx->cu->find_function(functionName); return function->getSchema().arguments().size(); } -extern "C" TorchScriptFunctionArgumentType torchScript_FunctionArgumentTypeByFunctionName(void *scriptCtx, const char* functionName, - size_t arg_index) { +extern "C" TorchScriptFunctionArgumentType +torchScript_FunctionArgumentTypeByFunctionName(void *scriptCtx, const char *functionName, + size_t arg_index) { ModuleContext *ctx = (ModuleContext *)scriptCtx; - torch::jit::Function* function = ctx->cu->find_function(functionName); + torch::jit::Function *function = ctx->cu->find_function(functionName); return getArgumentType(function->getSchema().arguments()[arg_index]); } -extern "C" bool torchScript_FunctionExists(void *scriptCtx, const char* functionName) { +extern "C" bool torchScript_FunctionExists(void *scriptCtx, const char *functionName) { ModuleContext *ctx = (ModuleContext *)scriptCtx; - torch::jit::Function* function = ctx->cu->find_function(functionName); - return function!=nullptr; + torch::jit::Function *function = ctx->cu->find_function(functionName); + return function != nullptr; } diff --git a/src/backends/libtorch_c/torch_extensions/torch_redis.cpp b/src/backends/libtorch_c/torch_extensions/torch_redis.cpp index 912cea2a0..5ac278a15 100644 --- a/src/backends/libtorch_c/torch_extensions/torch_redis.cpp +++ b/src/backends/libtorch_c/torch_extensions/torch_redis.cpp @@ -8,70 +8,70 @@ static torch::RegisterOperators registry; void registerRedisOps(void) { registry = torch::RegisterOperators("redis::execute", &redisExecute) - .op("redis::asList", &asList) - .op("redisAI::model_execute", &modelExecute); + .op("redis::asList", &asList) + .op("redisAI::model_execute", &modelExecute); } -torch::IValue IValueFromRedisReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply){ +torch::IValue IValueFromRedisReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply) { int reply_type = RedisModule_CallReplyType(reply); - switch(reply_type) { - case REDISMODULE_REPLY_NULL: { - return torch::IValue(); - } - case REDISMODULE_REPLY_STRING: { - size_t len; - const char *replyStr = RedisModule_CallReplyStringPtr(reply, &len); - std::string str = replyStr; - return torch::IValue(str.substr(0,len)); - } - case REDISMODULE_REPLY_INTEGER: { - int intValue = (int)RedisModule_CallReplyInteger(reply); - return torch::IValue(intValue); - } - case REDISMODULE_REPLY_ARRAY: { - c10::List vec(c10::AnyType::get()); - size_t len = RedisModule_CallReplyLength(reply); - for (auto i = 0; i < len; ++i) { - RedisModuleCallReply *subReply = RedisModule_CallReplyArrayElement(reply, i); - torch::IValue value = IValueFromRedisReply(ctx, subReply); - vec.push_back(value); - } - return torch::IValue(vec); - } - case REDISMODULE_REPLY_ERROR: { - size_t len; - const char *replyStr = RedisModule_CallReplyStringPtr(reply, &len); - std::string error_str = "Redis command returned an error: "+std::string(replyStr); - RedisModule_FreeCallReply(reply); - RedisModule_FreeThreadSafeContext(ctx); - throw std::runtime_error(error_str); - } - case REDISMODULE_REPLY_UNKNOWN: { - std::string error_str = "Redis command returned an error: "+std::string(strerror(errno)); - RedisModule_FreeThreadSafeContext(ctx); - throw(std::runtime_error(error_str)); - } - default:{ - RedisModule_FreeThreadSafeContext(ctx); - throw(std::runtime_error("Unexpected internal error")); + switch (reply_type) { + case REDISMODULE_REPLY_NULL: { + return torch::IValue(); + } + case REDISMODULE_REPLY_STRING: { + size_t len; + const char *replyStr = RedisModule_CallReplyStringPtr(reply, &len); + std::string str = replyStr; + return torch::IValue(str.substr(0, len)); + } + case REDISMODULE_REPLY_INTEGER: { + int intValue = (int)RedisModule_CallReplyInteger(reply); + return torch::IValue(intValue); + } + case REDISMODULE_REPLY_ARRAY: { + c10::List vec(c10::AnyType::get()); + size_t len = RedisModule_CallReplyLength(reply); + for (auto i = 0; i < len; ++i) { + RedisModuleCallReply *subReply = RedisModule_CallReplyArrayElement(reply, i); + torch::IValue value = IValueFromRedisReply(ctx, subReply); + vec.push_back(value); } + return torch::IValue(vec); + } + case REDISMODULE_REPLY_ERROR: { + size_t len; + const char *replyStr = RedisModule_CallReplyStringPtr(reply, &len); + std::string error_str = "Redis command returned an error: " + std::string(replyStr); + RedisModule_FreeCallReply(reply); + RedisModule_FreeThreadSafeContext(ctx); + throw std::runtime_error(error_str); + } + case REDISMODULE_REPLY_UNKNOWN: { + std::string error_str = "Redis command returned an error: " + std::string(strerror(errno)); + RedisModule_FreeThreadSafeContext(ctx); + throw(std::runtime_error(error_str)); + } + default: { + RedisModule_FreeThreadSafeContext(ctx); + throw(std::runtime_error("Unexpected internal error")); + } } } -torch::IValue redisExecute(const std::string& fn_name, const std::vector &args ) { - RedisModuleCtx* ctx = RedisModule_GetThreadSafeContext(nullptr); +torch::IValue redisExecute(const std::string &fn_name, const std::vector &args) { + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(nullptr); RedisModule_ThreadSafeContextLock(ctx); size_t len = args.size(); - RedisModuleString* arguments[len]; + RedisModuleString *arguments[len]; len = 0; for (auto &arg : args) { - const char* str = arg.c_str(); + const char *str = arg.c_str(); arguments[len++] = RedisModule_CreateString(ctx, str, strlen(str)); } RedisModuleCallReply *reply = RedisModule_Call(ctx, fn_name.c_str(), "!v", arguments, len); RedisModule_ThreadSafeContextUnlock(ctx); - for(int i= 0; i < len; i++){ + for (int i = 0; i < len; i++) { RedisModule_FreeString(nullptr, arguments[i]); } torch::IValue value = IValueFromRedisReply(ctx, reply); @@ -80,17 +80,17 @@ torch::IValue redisExecute(const std::string& fn_name, const std::vector asList(const torch::IValue &v) { - return v.toList(); -} +torch::List asList(const torch::IValue &v) { return v.toList(); } -std::vector modelExecute(const std::string& model_key, const std::vector &inputs, int64_t num_outputs) { - RedisModuleCtx* ctx = RedisModule_GetThreadSafeContext(nullptr); +std::vector modelExecute(const std::string &model_key, + const std::vector &inputs, + int64_t num_outputs) { + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(nullptr); // Prepare for getting model from key space. - const char* model_key_str = model_key.c_str(); - RedisModuleString *model_key_rs = RedisModule_CreateString(ctx, model_key_str, - model_key.size()); + const char *model_key_str = model_key.c_str(); + RedisModuleString *model_key_rs = + RedisModule_CreateString(ctx, model_key_str, model_key.size()); RAI_Error *err; RedisAI_InitError(&err); RAI_Model *model = nullptr; @@ -98,8 +98,7 @@ std::vector modelExecute(const std::string& model_key, const std: std::vector outputs; RedisModule_ThreadSafeContextLock(ctx); - int status = RedisAI_GetModelFromKeyspace(ctx, model_key_rs, &model, - REDISMODULE_READ, err); + int status = RedisAI_GetModelFromKeyspace(ctx, model_key_rs, &model, REDISMODULE_READ, err); RedisModule_FreeString(nullptr, model_key_rs); if (status != REDISMODULE_OK) { RedisModule_ThreadSafeContextUnlock(ctx); @@ -135,7 +134,7 @@ std::vector modelExecute(const std::string& model_key, const std: outputs.push_back(output); } - finish: +finish: if (model_run_ctx) { RedisAI_ModelRunCtxFree(model_run_ctx); } diff --git a/src/backends/onnx_allocator/onnx_allocator.cpp b/src/backends/onnx_allocator/onnx_allocator.cpp index 432db4893..2793319e3 100644 --- a/src/backends/onnx_allocator/onnx_allocator.cpp +++ b/src/backends/onnx_allocator/onnx_allocator.cpp @@ -6,41 +6,46 @@ struct RAIOrtAllocator : OrtAllocator { RAIOrtAllocator(); ~RAIOrtAllocator(); - RAIOrtAllocator(const RAIOrtAllocator&) = delete; - RAIOrtAllocator& operator=(const RAIOrtAllocator&) = delete; + RAIOrtAllocator(const RAIOrtAllocator &) = delete; + RAIOrtAllocator &operator=(const RAIOrtAllocator &) = delete; - void* Alloc(size_t size); - void Free(void* p); - const OrtMemoryInfo* Info() const; + void *Alloc(size_t size); + void Free(void *p); + const OrtMemoryInfo *Info() const; unsigned long long NumAllocatorAccess() const; unsigned long long MemoryInUse() const; void SetMemoryLimit(unsigned long long max_memory); static RAIOrtAllocator *GetInstance(); -private: + private: std::atomic memory_inuse{0}; std::atomic num_allocator_access{0}; unsigned long long memory_limit = 0; - OrtMemoryInfo* cpu_memory_info; - static RAIOrtAllocator* allocator_instance; + OrtMemoryInfo *cpu_memory_info; + static RAIOrtAllocator *allocator_instance; }; -RAIOrtAllocator* RAIOrtAllocator::allocator_instance = nullptr; +RAIOrtAllocator *RAIOrtAllocator::allocator_instance = nullptr; RAIOrtAllocator::RAIOrtAllocator() { OrtAllocator::version = ORT_API_VERSION; - OrtAllocator::Alloc = [](OrtAllocator* this_, size_t size) { return static_cast(this_)->Alloc(size); }; - OrtAllocator::Free = [](OrtAllocator* this_, void* p) { static_cast(this_)->Free(p); }; - OrtAllocator::Info = [](const OrtAllocator* this_) { return static_cast(this_)->Info(); }; - Ort::ThrowOnError(Ort::GetApi().CreateCpuMemoryInfo(OrtDeviceAllocator, OrtMemTypeDefault, &cpu_memory_info)); + OrtAllocator::Alloc = [](OrtAllocator *this_, size_t size) { + return static_cast(this_)->Alloc(size); + }; + OrtAllocator::Free = [](OrtAllocator *this_, void *p) { + static_cast(this_)->Free(p); + }; + OrtAllocator::Info = [](const OrtAllocator *this_) { + return static_cast(this_)->Info(); + }; + Ort::ThrowOnError( + Ort::GetApi().CreateCpuMemoryInfo(OrtDeviceAllocator, OrtMemTypeDefault, &cpu_memory_info)); RAIOrtAllocator::allocator_instance = this; } -RAIOrtAllocator::~RAIOrtAllocator() { - Ort::GetApi().ReleaseMemoryInfo(cpu_memory_info); -} +RAIOrtAllocator::~RAIOrtAllocator() { Ort::GetApi().ReleaseMemoryInfo(cpu_memory_info); } -void* RAIOrtAllocator::Alloc(size_t size) { +void *RAIOrtAllocator::Alloc(size_t size) { // Allocate an additional 63 bytes to ensure that we can return an address which is // 64-byte aligned, and an additional space in the size of a pointer to store // the address that RedisModule_Alloc returns. @@ -52,18 +57,21 @@ void* RAIOrtAllocator::Alloc(size_t size) { size_t cur_memory = memory_inuse.load(); if (memory_limit && cur_memory + allocated_size > memory_limit) { RedisModule_Free(allocated_address); - throw Ort::Exception("Onnxruntime memory limit exceeded, memory allocation failed.", ORT_RUNTIME_EXCEPTION); + throw Ort::Exception("Onnxruntime memory limit exceeded, memory allocation failed.", + ORT_RUNTIME_EXCEPTION); } memory_inuse.fetch_add(allocated_size); num_allocator_access.fetch_add(1); - // This operation guarantees that "aligned_address" is the closest 64-aligned address to ("allocated_address"+size_t). + // This operation guarantees that "aligned_address" is the closest 64-aligned address to + // ("allocated_address"+size_t). void **aligned_address = (void **)(((size_t)(allocated_address) + offset) & (~63)); - // This stores the address "allocated_address" right before "aligned_address" (so we can retrieve it when we free). + // This stores the address "allocated_address" right before "aligned_address" (so we can + // retrieve it when we free). aligned_address[-1] = allocated_address; return aligned_address; } -void RAIOrtAllocator::Free(void* p) { +void RAIOrtAllocator::Free(void *p) { if (p == nullptr) { return; } @@ -78,26 +86,20 @@ void RAIOrtAllocator::Free(void* p) { RedisModule_Free(allocated_address); } -const OrtMemoryInfo* RAIOrtAllocator::Info() const { - return cpu_memory_info; -} +const OrtMemoryInfo *RAIOrtAllocator::Info() const { return cpu_memory_info; } unsigned long long RAIOrtAllocator::NumAllocatorAccess() const { return num_allocator_access.load(); } -unsigned long long RAIOrtAllocator::MemoryInUse() const { - return memory_inuse.load(); -} +unsigned long long RAIOrtAllocator::MemoryInUse() const { return memory_inuse.load(); } void RAIOrtAllocator::SetMemoryLimit(unsigned long long max_memory) { // max_memory is given in MB - memory_limit = 1000000*max_memory; + memory_limit = 1000000 * max_memory; } -RAIOrtAllocator *RAIOrtAllocator::GetInstance() { - return RAIOrtAllocator::allocator_instance; -} +RAIOrtAllocator *RAIOrtAllocator::GetInstance() { return RAIOrtAllocator::allocator_instance; } OrtAllocator *CreateCustomAllocator(unsigned long long max_memory) { auto *allocator = new RAIOrtAllocator(); @@ -105,9 +107,7 @@ OrtAllocator *CreateCustomAllocator(unsigned long long max_memory) { return allocator; } -unsigned long long RAI_GetMemoryInfoORT() { - return RAIOrtAllocator::GetInstance()->MemoryInUse(); -} +unsigned long long RAI_GetMemoryInfoORT() { return RAIOrtAllocator::GetInstance()->MemoryInUse(); } unsigned long long RAI_GetMemoryAccessORT() { return RAIOrtAllocator::GetInstance()->NumAllocatorAccess();