From 29fc0039ff2b943e7cfa28d0919c522abe27eb00 Mon Sep 17 00:00:00 2001 From: Raaj Date: Sat, 12 Jan 2019 08:39:05 +0800 Subject: [PATCH] Python Pybind11 wrapper (#1014) --- .gitmodules | 3 + 3rdparty/pybind11 | 1 + CMakeLists.txt | 26 +- doc/modules/python_module.md | 22 +- .../tutorial_api_python/1_body_from_image.py | 67 ++ .../tutorial_api_python/1_extract_pose.py | 48 -- .../2_whole_body_from_image.py | 72 ++ .../3_heatmaps_from_image.py | 83 +++ examples/tutorial_api_python/CMakeLists.txt | 5 +- .../tutorial_api_python/openpose_python.py | 55 ++ examples/tutorial_developer/CMakeLists.txt | 3 - .../python_1_pose_from_heatmaps.py | 101 --- python/openpose/CMakeLists.txt | 10 +- python/openpose/__init__.py | 2 +- python/openpose/_openpose.cpp | 682 ++++++++++-------- python/openpose/openpose.py | 243 ------- scripts/osx/install_deps.sh | 2 + scripts/travis/configure_cmake.sh | 7 +- scripts/travis/run_tests.sh | 2 +- scripts/ubuntu/install_ubuntu_deps.sh | 4 +- 20 files changed, 699 insertions(+), 739 deletions(-) create mode 160000 3rdparty/pybind11 create mode 100644 examples/tutorial_api_python/1_body_from_image.py delete mode 100644 examples/tutorial_api_python/1_extract_pose.py create mode 100644 examples/tutorial_api_python/2_whole_body_from_image.py create mode 100644 examples/tutorial_api_python/3_heatmaps_from_image.py create mode 100644 examples/tutorial_api_python/openpose_python.py delete mode 100644 examples/tutorial_developer/python_1_pose_from_heatmaps.py delete mode 100644 python/openpose/openpose.py diff --git a/.gitmodules b/.gitmodules index bba100f72..4e1c0d5f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "3rdparty/caffe"] path = 3rdparty/caffe url = https://github.com/CMU-Perceptual-Computing-Lab/caffe.git +[submodule "3rdparty/pybind11"] + path = 3rdparty/pybind11 + url = https://github.com/pybind/pybind11.git diff --git a/3rdparty/pybind11 b/3rdparty/pybind11 new file mode 160000 index 000000000..111b25b26 --- /dev/null +++ b/3rdparty/pybind11 @@ -0,0 +1 @@ +Subproject commit 111b25b260fd687df9524d850c9d91927a661623 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a58f4141..bbccfc186 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -639,7 +639,7 @@ if (UNIX OR APPLE) file(GLOB CAFFE_DIR_VALID ${CMAKE_SOURCE_DIR}/3rdparty/caffe/*) list(LENGTH CAFFE_DIR_VALID CAFFE_DIR_VALID_LENGTH) if (CAFFE_DIR_VALID_LENGTH EQUAL 0) - execute_process(COMMAND git submodule update --init --recursive) + execute_process(COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/3rdparty/caffe) else (CAFFE_DIR_VALID_LENGTH EQUAL 0) message(STATUS "Caffe has already been downloaded.") endif (CAFFE_DIR_VALID_LENGTH EQUAL 0) @@ -713,7 +713,7 @@ if (UNIX OR APPLE) -DCPU_ONLY=${CAFFE_CPU_ONLY} -DCMAKE_BUILD_TYPE=Release -DBUILD_docs=OFF - -DBUILD_python=${BUILD_PYTHON} + -DBUILD_python=OFF -DBUILD_python_layer=OFF -DUSE_LEVELDB=OFF -DUSE_LMDB=OFF @@ -731,7 +731,7 @@ if (UNIX OR APPLE) -DCPU_ONLY=${CAFFE_CPU_ONLY} -DCMAKE_BUILD_TYPE=Release -DBUILD_docs=OFF - -DBUILD_python=${BUILD_PYTHON} + -DBUILD_python=OFF -DBUILD_python_layer=OFF -DUSE_LEVELDB=OFF -DUSE_LMDB=OFF @@ -912,12 +912,24 @@ download_model("hand" ${DOWNLOAD_HAND_MODEL} hand/pose_iter_102000.caffemodel message(STATUS "Models Downloaded.") + ### PYTHON -if (Caffe_FOUND) - if(BUILD_PYTHON) +if (BUILD_PYTHON) + if (WIN32) + execute_process(COMMAND cmd /c cd ${CMAKE_SOURCE_DIR} & git submodule update --init 3rdparty/pybind11/) + add_subdirectory(3rdparty/pybind11) add_subdirectory(python) - endif (BUILD_PYTHON) -endif(Caffe_FOUND) + elseif (UNIX OR APPLE) + if (Caffe_FOUND) + execute_process(COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/3rdparty/pybind11/) + add_subdirectory(3rdparty/pybind11) + add_subdirectory(python) + endif (Caffe_FOUND) + else (WIN32) + message(FATAL_ERROR "Unknown OS.") + endif (WIN32) +endif (BUILD_PYTHON) + ### GENERATE DOCUMENTATION if (UNIX OR APPLE) diff --git a/doc/modules/python_module.md b/doc/modules/python_module.md index 496ff9a84..34eaa8161 100644 --- a/doc/modules/python_module.md +++ b/doc/modules/python_module.md @@ -11,14 +11,14 @@ OpenPose Python Module and Demo ## Introduction -This experimental module exposes a Python API for OpenPose. This allows you to construct an OpenPose object, pass in a numpy array for an image, and get a numpy array of the pose positions. This API also exposes an API that allows you to directly pass in heatmaps from a network and extract poses out of it (Requires Python Caffe to be installed seperately) - -At present the Python API only supports body pose. Hands and Face will be added in the future. +This module exposes a Python API for OpenPose. It is effectively a wrapper that replicates most of the functionality of the [op::Wrapper class](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/include/openpose/wrapper/wrapper.hpp) and allows you to populate and retrieve data from the [op::Datum class](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/include/openpose/core/datum.hpp) using standard Python and Numpy constructs. ## Compatibility -The OpenPose Python module is compatible with both Python 2 and Python 3. In addition, it will also run in all OpenPose compatible operating systems. +The OpenPose Python module is compatible with both Python 2 and Python 3. In addition, it will also run in all OpenPose compatible operating systems. It uses [Pybind11](https://github.com/pybind/pybind11) for mapping between C++ and Python datatypes. + +To compile, enable `BUILD_PYTHON` in cmake. Pybind selects the latest version of Python by default (Python 3). To use Python 2, change `PYTHON_EXECUTABLE` and `PYTHON_LIBRARY` flags in cmake to your desired python version. @@ -28,22 +28,18 @@ Check [doc/installation.md#python-module](../installation.md#python-api) for ins The Python API requires Numpy for array management, and OpenCV for image loading. They can be installed via: ``` -pip install numpy -pip install opencv-python +pip install numpy opencv-python ``` ## Testing -Two examples can be found in `build/examples/tutorial_api_python` in your build folder. Navigate directly to this path to run examples. - -- `1_extract_pose` demonstrates a simple use of the API. -- `2_pose_from_heatmaps` demonstrates constructing pose from heatmaps from the caffe network (Requires Python Caffe to be installed seperately, only tested on Ubuntu). +All the Python examples from the Tutorial API Python module can be found in `build/examples/tutorial_api_python` in your build folder. Navigate directly to this path to run examples. ``` # From command line cd build/examples/tutorial_api_python -python 1_extract_pose.py +python3 1_body_from_image.py ``` @@ -51,5 +47,5 @@ python 1_extract_pose.py ## Exporting Python OpenPose Note: This step is only required if you are moving the `*.py` files outside their original location, or writting new `*.py` scripts outside `build/examples/tutorial_api_python`. -- Option a, installing OpenPose: On an Ubuntu or OSX based system, you could install OpenPose by running `sudo make install`, you could then set the OpenPose path in your python scripts to the OpenPose installation path (default: `/usr/local/python`) and start using OpenPose at any location. Take a look at `build/examples/tutorial_pose/1_extract_pose.py` for an example. -- Option b, not installing OpenPose: To move the OpenPose Python API demos to a different folder, ensure that the line `sys.path.append('{OpenPose_path}/python')` is properly set in your `*.py` files, where `{OpenPose_path}` points to your build folder of OpenPose. Take a look at `build/examples/tutorial_pose/1_extract_pose.py` for an example. +- Option a, installing OpenPose: On an Ubuntu or OSX based system, you could install OpenPose by running `sudo make install`, you could then set the OpenPose path in your python scripts to the OpenPose installation path (default: `/usr/local/python`) and start using OpenPose at any location. Take a look at `build/examples/tutorial_pose/1_body_from_image.py` for an example. +- Option b, not installing OpenPose: To move the OpenPose Python API demos to a different folder, ensure that the line `sys.path.append('{OpenPose_path}/python')` is properly set in your `*.py` files, where `{OpenPose_path}` points to your build folder of OpenPose. Take a look at `build/examples/tutorial_pose/1_body_from_image.py` for an example. diff --git a/examples/tutorial_api_python/1_body_from_image.py b/examples/tutorial_api_python/1_body_from_image.py new file mode 100644 index 000000000..544e0089f --- /dev/null +++ b/examples/tutorial_api_python/1_body_from_image.py @@ -0,0 +1,67 @@ +# From Python +# It requires OpenCV installed for Python +import sys +import cv2 +import os +from sys import platform +import argparse + +# Import Openpose (Windows/Ubuntu/OSX) +dir_path = os.path.dirname(os.path.realpath(__file__)) +try: + # Windows Import + if platform == "win32": + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append(dir_path + '/../../python/openpose/Release'); + os.environ['PATH'] = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;' + import _openpose as op + else: + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append('../../python'); + # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. + # sys.path.append('/usr/local/python') + from openpose import openpose as op +except: + raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') + +# Flags +parser = argparse.ArgumentParser() +parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).") +args = parser.parse_known_args() + +# Custom Params (refer to include/openpose/flags.hpp for more parameters) +params = dict() +params["model_folder"] = "../../../models/" + +# Add others in path? +for i in range(0, len(args[1])): + curr_item = args[1][i] + if i != len(args[1])-1: next_item = args[1][i+1] + else: next_item = "1" + if "--" in curr_item and "--" in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = "1" + elif "--" in curr_item and "--" not in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = next_item + +# Construct it from system arguments +# op.init_argv(args[1]) +# oppython = op.OpenposePython() + +# Starting OpenPose +opWrapper = op.WrapperPython() +opWrapper.configure(params) +opWrapper.start() + +# Process Image +datum = op.Datum() +imageToProcess = cv2.imread(args[0].image_path) +datum.cvInputData = imageToProcess +opWrapper.emplaceAndPop([datum]) + +# Display Image +print("Body keypoints: \n" + str(datum.poseKeypoints)) +while 1: + cv2.imshow("win", datum.cvOutputData) + cv2.waitKey(15) diff --git a/examples/tutorial_api_python/1_extract_pose.py b/examples/tutorial_api_python/1_extract_pose.py deleted file mode 100644 index b4ab0fd07..000000000 --- a/examples/tutorial_api_python/1_extract_pose.py +++ /dev/null @@ -1,48 +0,0 @@ -# From Python -# It requires OpenCV installed for Python -import sys -import cv2 -import os -from sys import platform - -# Remember to add your installation path here -# Option a -dir_path = os.path.dirname(os.path.realpath(__file__)) -if platform == "win32": sys.path.append(dir_path + '/../../python/openpose/'); -else: sys.path.append('../../python'); -# Option b -# If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. -# sys.path.append('/usr/local/python') - -# Parameters for OpenPose. Take a look at C++ OpenPose example for meaning of components. Ensure all below are filled -try: - from openpose import * -except: - raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') -params = dict() -params["logging_level"] = 3 -params["output_resolution"] = "-1x-1" -params["net_resolution"] = "-1x368" -params["model_pose"] = "BODY_25" -params["alpha_pose"] = 0.6 -params["scale_gap"] = 0.25 -params["scale_number"] = 1 -params["render_threshold"] = 0.05 -# If GPU version is built, and multiple GPUs are available, set the ID here -params["num_gpu_start"] = 0 -params["disable_blending"] = False -# Ensure you point to the correct path where models are located -params["default_model_folder"] = dir_path + "/../../../models/" -# Construct OpenPose object allocates GPU memory -openpose = OpenPose(params) - -while 1: - # Read new image - img = cv2.imread("../../../examples/media/COCO_val2014_000000000192.jpg") - # Output keypoints and the image with the human skeleton blended on it - keypoints, output_image = openpose.forward(img, True) - # Print the human pose keypoints, i.e., a [#people x #keypoints x 3]-dimensional numpy object with the keypoints of all the people on that image - print(keypoints) - # Display the image - cv2.imshow("output", output_image) - cv2.waitKey(15) diff --git a/examples/tutorial_api_python/2_whole_body_from_image.py b/examples/tutorial_api_python/2_whole_body_from_image.py new file mode 100644 index 000000000..40ce9d154 --- /dev/null +++ b/examples/tutorial_api_python/2_whole_body_from_image.py @@ -0,0 +1,72 @@ +# From Python +# It requires OpenCV installed for Python +import sys +import cv2 +import os +from sys import platform +import argparse + +# Import Openpose (Windows/Ubuntu/OSX) +dir_path = os.path.dirname(os.path.realpath(__file__)) +try: + # Windows Import + if platform == "win32": + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append(dir_path + '/../../python/openpose/Release'); + os.environ['PATH'] = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;' + import _openpose as op + else: + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append('../../python'); + # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. + # sys.path.append('/usr/local/python') + from openpose import openpose as op +except: + raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') + +# Flags +parser = argparse.ArgumentParser() +parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000241.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).") +args = parser.parse_known_args() + +# Custom Params (refer to include/openpose/flags.hpp for more parameters) +params = dict() +params["model_folder"] = "../../../models/" +params["face"] = True +params["hand"] = True + +# Add others in path? +for i in range(0, len(args[1])): + curr_item = args[1][i] + if i != len(args[1])-1: next_item = args[1][i+1] + else: next_item = "1" + if "--" in curr_item and "--" in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = "1" + elif "--" in curr_item and "--" not in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = next_item + +# Construct it from system arguments +# op.init_argv(args[1]) +# oppython = op.OpenposePython() + +# Starting OpenPose +opWrapper = op.WrapperPython() +opWrapper.configure(params) +opWrapper.start() + +# Process Image +datum = op.Datum() +imageToProcess = cv2.imread(args[0].image_path) +datum.cvInputData = imageToProcess +opWrapper.emplaceAndPop([datum]) + +# Display Image +print("Body keypoints: \n" + str(datum.poseKeypoints)) +print("Face keypoints: \n" + str(datum.faceKeypoints)) +print("Left hand keypoints: \n" + str(datum.handKeypoints[0])) +print("Right hand keypoints: \n" + str(datum.handKeypoints[1])) +while 1: + cv2.imshow("win", datum.cvOutputData) + cv2.waitKey(15) diff --git a/examples/tutorial_api_python/3_heatmaps_from_image.py b/examples/tutorial_api_python/3_heatmaps_from_image.py new file mode 100644 index 000000000..675f564af --- /dev/null +++ b/examples/tutorial_api_python/3_heatmaps_from_image.py @@ -0,0 +1,83 @@ +# From Python +# It requires OpenCV installed for Python +import sys +import cv2 +import os +from sys import platform +import argparse +import numpy as np + +# Import Openpose (Windows/Ubuntu/OSX) +dir_path = os.path.dirname(os.path.realpath(__file__)) +try: + # Windows Import + if platform == "win32": + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append(dir_path + '/../../python/openpose/Release'); + os.environ['PATH'] = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;' + import _openpose as op + else: + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append('../../python'); + # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. + # sys.path.append('/usr/local/python') + from openpose import openpose as op +except: + raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') + +# Flags +parser = argparse.ArgumentParser() +parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).") +args = parser.parse_known_args() + +# Custom Params (refer to include/openpose/flags.hpp for more parameters) +params = dict() +params["model_folder"] = "../../../models/" +params["heatmaps_add_parts"] = True +params["heatmaps_add_PAFs"] = True + +# Add others in path? +for i in range(0, len(args[1])): + curr_item = args[1][i] + if i != len(args[1])-1: next_item = args[1][i+1] + else: next_item = "1" + if "--" in curr_item and "--" in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = "1" + elif "--" in curr_item and "--" not in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = next_item + +# Construct it from system arguments +# op.init_argv(args[1]) +# oppython = op.OpenposePython() + +# Starting OpenPose +opWrapper = op.WrapperPython() +opWrapper.configure(params) +opWrapper.start() + +# Process Image +datum = op.Datum() +imageToProcess = cv2.imread(args[0].image_path) +datum.cvInputData = imageToProcess +opWrapper.emplaceAndPop([datum]) + +# Process outputs +outputImageF = (datum.inputNetData[0].copy())[0,:,:,:] + 0.5 +outputImageF = cv2.merge([outputImageF[0,:,:], outputImageF[1,:,:], outputImageF[2,:,:]]) +outputImageF = (outputImageF*255.).astype(dtype='uint8') +heatmaps = datum.poseHeatMaps.copy() +heatmaps = (heatmaps*255.).astype(dtype='uint8') + +# Display Image +counter = 0 +while 1: + num_maps = heatmaps.shape[0] + heatmap = heatmaps[counter, :, :].copy() + heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) + combined = cv2.addWeighted(outputImageF, 0.5, heatmap, 0.5, 0) + cv2.imshow("win", combined) + cv2.waitKey(-1) + counter += 1 + counter = counter % num_maps diff --git a/examples/tutorial_api_python/CMakeLists.txt b/examples/tutorial_api_python/CMakeLists.txt index 437814b14..0ee5be182 100644 --- a/examples/tutorial_api_python/CMakeLists.txt +++ b/examples/tutorial_api_python/CMakeLists.txt @@ -1,2 +1,5 @@ ### Add Python Test -configure_file(1_extract_pose.py 1_extract_pose.py) +configure_file(openpose_python.py openpose_python.py) +configure_file(1_body_from_image.py 1_body_from_image.py) +configure_file(2_whole_body_from_image.py 2_whole_body_from_image.py) +configure_file(3_heatmaps_from_image.py 3_heatmaps_from_image.py) diff --git a/examples/tutorial_api_python/openpose_python.py b/examples/tutorial_api_python/openpose_python.py new file mode 100644 index 000000000..939a846ef --- /dev/null +++ b/examples/tutorial_api_python/openpose_python.py @@ -0,0 +1,55 @@ +# From Python +# It requires OpenCV installed for Python +import sys +import cv2 +import os +from sys import platform +import argparse + +# Import Openpose (Windows/Ubuntu/OSX) +dir_path = os.path.dirname(os.path.realpath(__file__)) +try: + # Windows Import + if platform == "win32": + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append(dir_path + '/../../python/openpose/Release'); + os.environ['PATH'] = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;' + import _openpose as op + else: + # Change these variables to point to the correct folder (Release/x64 etc.) + sys.path.append('../../python'); + # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. + # sys.path.append('/usr/local/python') + from openpose import openpose as op +except: + raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') + +# Flags +parser = argparse.ArgumentParser() +parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).") +args = parser.parse_known_args() + +# Custom Params (refer to include/openpose/flags.hpp for more parameters) +params = dict() +params["model_folder"] = "../../../models/" + +# Add others in path? +for i in range(0, len(args[1])): + curr_item = args[1][i] + if i != len(args[1])-1: next_item = args[1][i+1] + else: next_item = "1" + if "--" in curr_item and "--" in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = "1" + elif "--" in curr_item and "--" not in next_item: + key = curr_item.replace('-','') + if key not in params: params[key] = next_item + +# Construct it from system arguments +# op.init_argv(args[1]) +# oppython = op.OpenposePython() + +# Starting OpenPose +opWrapper = op.WrapperPython(3) +opWrapper.configure(params) +opWrapper.execute() diff --git a/examples/tutorial_developer/CMakeLists.txt b/examples/tutorial_developer/CMakeLists.txt index 6b66cd737..e4b1a3c34 100644 --- a/examples/tutorial_developer/CMakeLists.txt +++ b/examples/tutorial_developer/CMakeLists.txt @@ -30,6 +30,3 @@ foreach(EXAMPLE_FILE ${EXAMPLE_FILES}) endif (WIN32) endforeach() - -### Add Python files -configure_file(python_1_pose_from_heatmaps.py python_1_pose_from_heatmaps.py) diff --git a/examples/tutorial_developer/python_1_pose_from_heatmaps.py b/examples/tutorial_developer/python_1_pose_from_heatmaps.py deleted file mode 100644 index c822938a5..000000000 --- a/examples/tutorial_developer/python_1_pose_from_heatmaps.py +++ /dev/null @@ -1,101 +0,0 @@ -from sys import platform -import sys -try: - import caffe -except ImportError: - print("This sample can only be run if Python Caffe if available on your system") - print("Currently OpenPose does not compile Python Caffe. This may be supported in the future") - sys.exit(-1) - -import os -os.environ["GLOG_minloglevel"] = "1" -import caffe -import cv2 -import numpy as np -import sys -import time -dir_path = os.path.dirname(os.path.realpath(__file__)) -sys.path.append('../../python') -dir_path + "/../../models/" -try: - from openpose import OpenPose -except: - raise Exception('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') - -# Params for change -# Single-scale -defRes = 368 -scales = [1] -# # Multi-scale -# defRes = 736 -# scales = [1, 0.75, 0.5, 0.25] -class Param: - caffemodel = dir_path + "/../../../models/pose/body_25/pose_iter_584000.caffemodel" - prototxt = dir_path + "/../../../models/pose/body_25/pose_deploy.prototxt" - -# Load OpenPose object and Caffe Nets -params = dict() -params["logging_level"] = 3 -params["output_resolution"] = "-1x-1" -params["net_resolution"] = "-1x"+str(defRes) -params["model_pose"] = "BODY_25" -params["alpha_pose"] = 0.6 -params["scale_gap"] = 0.25 -params["scale_number"] = len(scales) -params["render_threshold"] = 0.05 -params["num_gpu_start"] = 0 -params["disable_blending"] = False -params["default_model_folder"] = dir_path + "/../../../models/" -openpose = OpenPose(params) -caffe.set_mode_gpu() -caffe.set_device(0) -nets = [] -for scale in scales: - nets.append(caffe.Net(Param.prototxt, Param.caffemodel, caffe.TEST)) -print("Net loaded") - -# Test Function -first_run = True -def func(frame): - - # Get image processed for network, and scaled image - imagesForNet, imagesOrig = OpenPose.process_frames(frame, defRes, scales) - - # Reshape - global first_run - if first_run: - for i in range(0, len(scales)): - net = nets[i] - imageForNet = imagesForNet[i] - in_shape = net.blobs['image'].data.shape - in_shape = (1, 3, imageForNet.shape[1], imageForNet.shape[2]) - net.blobs['image'].reshape(*in_shape) - net.reshape() - - first_run = False - print("Reshaped") - - # Forward pass to get heatmaps - heatmaps = [] - for i in range(0, len(scales)): - net = nets[i] - imageForNet = imagesForNet[i] - net.blobs['image'].data[0,:,:,:] = imageForNet - net.forward() - heatmaps.append(net.blobs['net_output'].data[:,:,:,:]) - - # Pose from HM Test - array, frame = openpose.poseFromHM(frame, heatmaps, scales) - - # Draw Heatmaps instead - #hm = heatmaps[0][:,0:18,:,:]; frame = OpenPose.draw_all(imagesOrig[0], hm, -1, 1, True) - #paf = heatmaps[0][:,20:,:,:]; frame = OpenPose.draw_all(imagesOrig[0], paf, -1, 4, False) - - return frame - - -img = cv2.imread(dir_path + "/../../../examples/media/COCO_val2014_000000000192.jpg") -frame = func(img) -while 1: - cv2.imshow("output", frame) - cv2.waitKey(15) diff --git a/python/openpose/CMakeLists.txt b/python/openpose/CMakeLists.txt index 280348fda..4fb705bf6 100644 --- a/python/openpose/CMakeLists.txt +++ b/python/openpose/CMakeLists.txt @@ -3,12 +3,12 @@ set(PYTHON_FILES __init__.py _openpose.cpp) -add_library(_openpose SHARED ${PYTHON_FILES}) -target_link_libraries(_openpose openpose ${OpenPose_3rdparty_libraries}) +pybind11_add_module(_openpose _openpose.cpp) + +target_link_libraries(_openpose PRIVATE pybind11::module openpose_src ${OpenPose_3rdparty_libraries}) SET_TARGET_PROPERTIES(_openpose PROPERTIES PREFIX "") -configure_file(openpose.py openpose.py) configure_file(__init__.py __init__.py) #install(TARGETS _openpose DESTINATION python) -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION python/openpose FILES_MATCHING PATTERN "*.so") -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION python/openpose FILES_MATCHING PATTERN "*.py") \ No newline at end of file +#install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION python/openpose FILES_MATCHING PATTERN "*.so") +#install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ DESTINATION python/openpose FILES_MATCHING PATTERN "*.py") diff --git a/python/openpose/__init__.py b/python/openpose/__init__.py index 60c2d2782..1ed8198a7 100644 --- a/python/openpose/__init__.py +++ b/python/openpose/__init__.py @@ -1 +1 @@ -from openpose import * +from . import _openpose as openpose diff --git a/python/openpose/_openpose.cpp b/python/openpose/_openpose.cpp index 0e5778179..ee3af76f5 100644 --- a/python/openpose/_openpose.cpp +++ b/python/openpose/_openpose.cpp @@ -2,24 +2,15 @@ #define OPENPOSE_PYTHON_HPP #define BOOST_DATE_TIME_NO_LIB -// OpenPose dependencies -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include + +#include +#include +#include +#include +#include #ifdef _WIN32 #define OP_EXPORT __declspec(dllexport) @@ -27,328 +18,391 @@ #define OP_EXPORT #endif -#define default_logging_level 3 -#define default_output_resolution "-1x-1" -#define default_net_resolution "-1x368" -#define default_model_pose "COCO" -#define default_alpha_pose 0.6 -#define default_scale_gap 0.25 -#define default_scale_number 1 -#define default_render_threshold 0.05 -#define default_num_gpu_start 0 -#define default_disable_blending false -#define default_model_folder "models/" - -// Todo, have GPU Number, handle, OpenCL/CPU Cases -OP_API class OpenPose { +namespace op{ + +namespace py = pybind11; + +void parse_gflags(const std::vector& argv) +{ + std::vector argv_vec; + for(auto& arg : argv) argv_vec.emplace_back((char*)arg.c_str()); + char** cast = &argv_vec[0]; + int size = argv_vec.size(); + gflags::ParseCommandLineFlags(&size, &cast, true); +} + +void init_int(py::dict d) +{ + std::vector argv; + argv.emplace_back("openpose.py"); + for (auto item : d){ + argv.emplace_back("--" + std::string(py::str(item.first))); + argv.emplace_back(py::str(item.second)); + } + parse_gflags(argv); +} + +void init_argv(std::vector argv) +{ + argv.insert(argv.begin(), "openpose.py"); + parse_gflags(argv); +} + +class WrapperPython{ public: - std::unique_ptr poseExtractorCaffe; - std::unique_ptr poseRenderer; - std::unique_ptr frameDisplayer; - std::unique_ptr scaleAndSizeExtractor; - - std::unique_ptr> resizeAndMergeCaffe; - std::unique_ptr> nmsCaffe; - std::unique_ptr> bodyPartConnectorCaffe; - std::shared_ptr> heatMapsBlob; - std::shared_ptr> peaksBlob; - op::Array mPoseKeypoints; - op::Array mPoseScores; - op::PoseModel poseModel; - int mGpuID; - - OpenPose(int FLAGS_logging_level = default_logging_level, - std::string FLAGS_output_resolution = default_output_resolution, - std::string FLAGS_net_resolution = default_net_resolution, - std::string FLAGS_model_pose = default_model_pose, - float FLAGS_alpha_pose = default_alpha_pose, - float FLAGS_scale_gap = default_scale_gap, - int FLAGS_scale_number = default_scale_number, - float FLAGS_render_threshold = default_render_threshold, - int FLAGS_num_gpu_start = default_num_gpu_start, - int FLAGS_disable_blending = default_disable_blending, - std::string FLAGS_model_folder = default_model_folder - ) { - mGpuID = FLAGS_num_gpu_start; -#ifdef USE_CUDA - caffe::Caffe::set_mode(caffe::Caffe::GPU); - caffe::Caffe::SetDevice(mGpuID); -#elif defined USE_OPENCL - caffe::Caffe::set_mode(caffe::Caffe::GPU); - std::vector devices; - const int maxNumberGpu = op::OpenCL::getTotalGPU(); - for (auto i = 0; i < maxNumberGpu; i++) - devices.emplace_back(i); - caffe::Caffe::SetDevices(devices); - caffe::Caffe::SelectDevice(mGpuID, true); - op::OpenCL::getInstance(mGpuID, CL_DEVICE_TYPE_GPU, true); -#else - caffe::Caffe::set_mode(caffe::Caffe::CPU); -#endif - op::log("OpenPose Library Python Wrapper", op::Priority::High); - // ------------------------- INITIALIZATION ------------------------- - // Step 1 - Set logging level - // - 0 will output all the logging messages - // - 255 will output nothing + std::unique_ptr opWrapper; + + WrapperPython(int mode = 0) + { + op::log("Starting OpenPose Python Wrapper...", op::Priority::High); + + // Construct opWrapper + opWrapper = std::unique_ptr(new op::Wrapper(static_cast(mode))); + } + + void configure(py::dict params = py::dict()) + { + if(params.size()) init_int(params); + + // logging_level + op::check(0 <= FLAGS_logging_level && FLAGS_logging_level <= 255, "Wrong logging_level value.", + __LINE__, __FUNCTION__, __FILE__); op::ConfigureLog::setPriorityThreshold((op::Priority)FLAGS_logging_level); - op::log("", op::Priority::Low, __LINE__, __FUNCTION__, __FILE__); - // Step 2 - Read GFlags (user defined configuration) + op::Profiler::setDefaultX(FLAGS_profile_speed); + + // Applying user defined configuration - GFlags to program variables // outputSize const auto outputSize = op::flagsToPoint(FLAGS_output_resolution, "-1x-1"); // netInputSize const auto netInputSize = op::flagsToPoint(FLAGS_net_resolution, "-1x368"); + // faceNetInputSize + const auto faceNetInputSize = op::flagsToPoint(FLAGS_face_net_resolution, "368x368 (multiples of 16)"); + // handNetInputSize + const auto handNetInputSize = op::flagsToPoint(FLAGS_hand_net_resolution, "368x368 (multiples of 16)"); // poseModel - poseModel = op::flagsToPoseModel(FLAGS_model_pose); - // Check no contradictory flags enabled - if (FLAGS_alpha_pose < 0. || FLAGS_alpha_pose > 1.) - op::error("Alpha value for blending must be in the range [0,1].", __LINE__, __FUNCTION__, __FILE__); - if (FLAGS_scale_gap <= 0. && FLAGS_scale_number > 1) - op::error("Incompatible flag configuration: scale_gap must be greater than 0 or scale_number = 1.", - __LINE__, __FUNCTION__, __FILE__); - // Step 3 - Initialize all required classes - scaleAndSizeExtractor = std::unique_ptr(new op::ScaleAndSizeExtractor(netInputSize, outputSize, FLAGS_scale_number, FLAGS_scale_gap)); - - poseExtractorCaffe = std::unique_ptr(new op::PoseExtractorCaffe{ poseModel, FLAGS_model_folder, FLAGS_num_gpu_start }); - - poseRenderer = std::unique_ptr(new op::PoseCpuRenderer{ poseModel, (float)FLAGS_render_threshold, !FLAGS_disable_blending, - (float)FLAGS_alpha_pose }); - frameDisplayer = std::unique_ptr(new op::FrameDisplayer{ "OpenPose Tutorial - Example 1", outputSize }); - - // Custom - resizeAndMergeCaffe = std::unique_ptr>(new op::ResizeAndMergeCaffe{}); - nmsCaffe = std::unique_ptr>(new op::NmsCaffe{}); - bodyPartConnectorCaffe = std::unique_ptr>(new op::BodyPartConnectorCaffe{}); - heatMapsBlob = { std::make_shared>(1,1,1,1) }; - peaksBlob = { std::make_shared>(1,1,1,1) }; - bodyPartConnectorCaffe->setPoseModel(poseModel); - - // Step 4 - Initialize resources on desired thread (in this case single thread, i.e., we init resources here) - poseExtractorCaffe->initializationOnThread(); - poseRenderer->initializationOnThread(); + const auto poseModel = op::flagsToPoseModel(FLAGS_model_pose); + // JSON saving + if (!FLAGS_write_keypoint.empty()) + op::log("Flag `write_keypoint` is deprecated and will eventually be removed." + " Please, use `write_json` instead.", op::Priority::Max); + // keypointScale + const auto keypointScale = op::flagsToScaleMode(FLAGS_keypoint_scale); + // heatmaps to add + const auto heatMapTypes = op::flagsToHeatMaps(FLAGS_heatmaps_add_parts, FLAGS_heatmaps_add_bkg, + FLAGS_heatmaps_add_PAFs); + const auto heatMapScale = op::flagsToHeatMapScaleMode(FLAGS_heatmaps_scale); + // >1 camera view? + const auto multipleView = (FLAGS_3d || FLAGS_3d_views > 1); + // Enabling Google Logging + const bool enableGoogleLogging = true; + + // Pose configuration (use WrapperStructPose{} for default and recommended configuration) + const op::WrapperStructPose wrapperStructPose{ + !FLAGS_body_disable, netInputSize, outputSize, keypointScale, FLAGS_num_gpu, FLAGS_num_gpu_start, + FLAGS_scale_number, (float)FLAGS_scale_gap, op::flagsToRenderMode(FLAGS_render_pose, multipleView), + poseModel, !FLAGS_disable_blending, (float)FLAGS_alpha_pose, (float)FLAGS_alpha_heatmap, + FLAGS_part_to_show, FLAGS_model_folder, heatMapTypes, heatMapScale, FLAGS_part_candidates, + (float)FLAGS_render_threshold, FLAGS_number_people_max, FLAGS_maximize_positives, FLAGS_fps_max, + FLAGS_prototxt_path, FLAGS_caffemodel_path, enableGoogleLogging}; + opWrapper->configure(wrapperStructPose); + // Face configuration (use op::WrapperStructFace{} to disable it) + const op::WrapperStructFace wrapperStructFace{ + FLAGS_face, faceNetInputSize, op::flagsToRenderMode(FLAGS_face_render, multipleView, FLAGS_render_pose), + (float)FLAGS_face_alpha_pose, (float)FLAGS_face_alpha_heatmap, (float)FLAGS_face_render_threshold}; + opWrapper->configure(wrapperStructFace); + // Hand configuration (use op::WrapperStructHand{} to disable it) + const op::WrapperStructHand wrapperStructHand{ + FLAGS_hand, handNetInputSize, FLAGS_hand_scale_number, (float)FLAGS_hand_scale_range, FLAGS_hand_tracking, + op::flagsToRenderMode(FLAGS_hand_render, multipleView, FLAGS_render_pose), (float)FLAGS_hand_alpha_pose, + (float)FLAGS_hand_alpha_heatmap, (float)FLAGS_hand_render_threshold}; + opWrapper->configure(wrapperStructHand); + // Extra functionality configuration (use op::WrapperStructExtra{} to disable it) + const op::WrapperStructExtra wrapperStructExtra{ + FLAGS_3d, FLAGS_3d_min_views, FLAGS_identification, FLAGS_tracking, FLAGS_ik_threads}; + opWrapper->configure(wrapperStructExtra); + // Output (comment or use default argument to disable any output) + const op::WrapperStructOutput wrapperStructOutput{ + FLAGS_cli_verbose, FLAGS_write_keypoint, op::stringToDataFormat(FLAGS_write_keypoint_format), + FLAGS_write_json, FLAGS_write_coco_json, FLAGS_write_coco_foot_json, FLAGS_write_coco_json_variant, + FLAGS_write_images, FLAGS_write_images_format, FLAGS_write_video, FLAGS_write_video_fps, + FLAGS_write_heatmaps, FLAGS_write_heatmaps_format, FLAGS_write_video_3d, FLAGS_write_video_adam, + FLAGS_write_bvh, FLAGS_udp_host, FLAGS_udp_port}; + opWrapper->configure(wrapperStructOutput); + // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); + // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) + if (FLAGS_disable_multi_thread) + opWrapper->disableMultiThreading(); } - std::vector*> caffeNetSharedToPtr( - std::vector>>& caffeNetOutputBlob) - { - try - { - // Prepare spCaffeNetOutputBlobss - std::vector*> caffeNetOutputBlobs(caffeNetOutputBlob.size()); - for (auto i = 0u; i < caffeNetOutputBlobs.size(); i++) - caffeNetOutputBlobs[i] = caffeNetOutputBlob[i].get(); - return caffeNetOutputBlobs; - } - catch (const std::exception& e) - { - op::error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return{}; - } + void start(){ + opWrapper->start(); } - void forward(const cv::Mat& inputImage, op::Array& poseKeypoints, cv::Mat& displayImage, bool display = false) { - op::OpOutputToCvMat opOutputToCvMat; - op::CvMatToOpInput cvMatToOpInput; - op::CvMatToOpOutput cvMatToOpOutput; - if (inputImage.empty()) - op::error("Could not open or find the image: ", __LINE__, __FUNCTION__, __FILE__); - const op::Point imageSize{ inputImage.cols, inputImage.rows }; - // Step 2 - Get desired scale sizes - std::vector scaleInputToNetInputs; - std::vector> netInputSizes; - double scaleInputToOutput; - op::Point outputResolution; - std::tie(scaleInputToNetInputs, netInputSizes, scaleInputToOutput, outputResolution) - = scaleAndSizeExtractor->extract(imageSize); - // Step 3 - Format input image to OpenPose input and output formats - const auto netInputArray = cvMatToOpInput.createArray(inputImage, scaleInputToNetInputs, netInputSizes); - - // Step 4 - Estimate poseKeypoints - poseExtractorCaffe->forwardPass(netInputArray, imageSize, scaleInputToNetInputs); - poseKeypoints = poseExtractorCaffe->getPoseKeypoints(); - - if (display) { - auto outputArray = cvMatToOpOutput.createArray(inputImage, scaleInputToOutput, outputResolution); - // Step 5 - Render poseKeypoints - poseRenderer->renderPose(outputArray, poseKeypoints, scaleInputToOutput); - // Step 6 - OpenPose output format to cv::Mat - displayImage = opOutputToCvMat.formatToCvMat(outputArray); - } + void stop(){ + opWrapper->stop(); } - void poseFromHeatmap(const cv::Mat& inputImage, std::vector>>& caffeNetOutputBlob, op::Array& poseKeypoints, cv::Mat& displayImage, std::vector>& imageSizes) { - // Get Scale - const op::Point inputDataSize{ inputImage.cols, inputImage.rows }; - - // Convert to Ptr - //std::vector>> a; - //caffeNetOutputBlob.emplace_back(caffeHmPtr); - const auto caffeNetOutputBlobs = caffeNetSharedToPtr(caffeNetOutputBlob); - - // To be called once only - resizeAndMergeCaffe->Reshape(caffeNetOutputBlobs, { heatMapsBlob.get() }, - op::getPoseNetDecreaseFactor(poseModel), 1.f / 1.f, true, - 0); - nmsCaffe->Reshape({ heatMapsBlob.get() }, { peaksBlob.get() }, op::getPoseMaxPeaks(), - op::getPoseNumberBodyParts(poseModel), 0); - bodyPartConnectorCaffe->Reshape({ heatMapsBlob.get(), peaksBlob.get() }); - - // Normal - op::OpOutputToCvMat opOutputToCvMat; - op::CvMatToOpInput cvMatToOpInput; - op::CvMatToOpOutput cvMatToOpOutput; - if (inputImage.empty()) - op::error("Could not open or find the image: ", __LINE__, __FUNCTION__, __FILE__); - const op::Point imageSize{ inputImage.cols, inputImage.rows }; - // Step 2 - Get desired scale sizes - std::vector scaleInputToNetInputs; - std::vector> netInputSizes; - double scaleInputToOutput; - op::Point outputResolution; - - std::tie(scaleInputToNetInputs, netInputSizes, scaleInputToOutput, outputResolution) - = scaleAndSizeExtractor->extract(imageSize); - - const auto netInputArray = cvMatToOpInput.createArray(inputImage, scaleInputToNetInputs, netInputSizes); - - // Run the modes - const std::vector floatScaleRatios(scaleInputToNetInputs.begin(), scaleInputToNetInputs.end()); - resizeAndMergeCaffe->setScaleRatios(floatScaleRatios); - std::vector*> heatMapsBlobs{ heatMapsBlob.get() }; - std::vector*> peaksBlobs{ peaksBlob.get() }; -#ifdef USE_CUDA - resizeAndMergeCaffe->Forward_gpu(caffeNetOutputBlobs, heatMapsBlobs); // ~5ms -#elif defined USE_OPENCL - resizeAndMergeCaffe->Forward_ocl(caffeNetOutputBlobs, heatMapsBlobs); // ~5ms -#else - resizeAndMergeCaffe->Forward_cpu(caffeNetOutputBlobs, heatMapsBlobs); // ~5ms -#endif - - nmsCaffe->setThreshold((float)poseExtractorCaffe->get(op::PoseProperty::NMSThreshold)); -#ifdef USE_CUDA - nmsCaffe->Forward_gpu(heatMapsBlobs, peaksBlobs);// ~2ms -#elif defined USE_OPENCL - nmsCaffe->Forward_ocl(heatMapsBlobs, peaksBlobs);// ~2ms -#else - nmsCaffe->Forward_cpu(heatMapsBlobs, peaksBlobs);// ~2ms -#endif - op::cudaCheck(__LINE__, __FUNCTION__, __FILE__); - - float mScaleNetToOutput = 1. / scaleInputToNetInputs[0]; - bodyPartConnectorCaffe->setScaleNetToOutput(mScaleNetToOutput); - bodyPartConnectorCaffe->setInterMinAboveThreshold( - (float)poseExtractorCaffe->get(op::PoseProperty::ConnectInterMinAboveThreshold) - ); - bodyPartConnectorCaffe->setInterThreshold((float)poseExtractorCaffe->get(op::PoseProperty::ConnectInterThreshold)); - bodyPartConnectorCaffe->setMinSubsetCnt((int)poseExtractorCaffe->get(op::PoseProperty::ConnectMinSubsetCnt)); - bodyPartConnectorCaffe->setMinSubsetScore((float)poseExtractorCaffe->get(op::PoseProperty::ConnectMinSubsetScore)); - -#ifdef USE_CUDA - bodyPartConnectorCaffe->Forward_gpu({ heatMapsBlob.get(), - peaksBlob.get() }, - mPoseKeypoints, mPoseScores); -#else - bodyPartConnectorCaffe->Forward_cpu({ heatMapsBlob.get(), - peaksBlob.get() }, - mPoseKeypoints, mPoseScores); -#endif - poseKeypoints = mPoseKeypoints; + void exec(){ + const auto cameraSize = op::flagsToPoint(FLAGS_camera_resolution, "-1x-1"); + op::ProducerType producerType; + std::string producerString; + std::tie(producerType, producerString) = op::flagsToProducer( + FLAGS_image_dir, FLAGS_video, FLAGS_ip_camera, FLAGS_camera, FLAGS_flir_camera, FLAGS_flir_camera_index); + // Producer (use default to disable any input) + const op::WrapperStructInput wrapperStructInput{ + producerType, producerString, FLAGS_frame_first, FLAGS_frame_step, FLAGS_frame_last, + FLAGS_process_real_time, FLAGS_frame_flip, FLAGS_frame_rotate, FLAGS_frames_repeat, + cameraSize, FLAGS_camera_parameter_path, FLAGS_frame_undistort, FLAGS_3d_views}; + opWrapper->configure(wrapperStructInput); + // GUI (comment or use default argument to disable any visual output) + const op::WrapperStructGui wrapperStructGui{ + op::flagsToDisplayMode(FLAGS_display, FLAGS_3d), !FLAGS_no_gui_verbose, FLAGS_fullscreen}; + opWrapper->configure(wrapperStructGui); + opWrapper->exec(); + } - auto outputArray = cvMatToOpOutput.createArray(inputImage, scaleInputToOutput, outputResolution); - // Step 5 - Render poseKeypoints - poseRenderer->renderPose(outputArray, mPoseKeypoints, scaleInputToOutput); - // Step 6 - OpenPose output format to cv::Mat - displayImage = opOutputToCvMat.formatToCvMat(outputArray); + void emplaceAndPop(std::vector>& l) + { + auto datumsPtr = std::make_shared>>(l); + opWrapper->emplaceAndPop(datumsPtr); } }; -#ifdef __cplusplus -extern "C" { -#endif +PYBIND11_MODULE(_openpose, m) { + + // Functions for Init Params + m.def("init_int", &init_int, "Init Function"); + m.def("init_argv", &init_argv, "Init Function"); + + // OpenposePython + py::class_(m, "WrapperPython") + .def(py::init<>()) + .def(py::init()) + .def("configure", &WrapperPython::configure) + .def("start", &WrapperPython::start) + .def("stop", &WrapperPython::stop) + .def("execute", &WrapperPython::exec) + .def("emplaceAndPop", &WrapperPython::emplaceAndPop) + ; + + // Datum Object + py::class_>(m, "Datum") + .def(py::init<>()) + .def_readwrite("id", &op::Datum::id) + .def_readwrite("subId", &op::Datum::subId) + .def_readwrite("subIdMax", &op::Datum::subIdMax) + .def_readwrite("name", &op::Datum::name) + .def_readwrite("frameNumber", &op::Datum::frameNumber) + .def_readwrite("cvInputData", &op::Datum::cvInputData) + .def_readwrite("inputNetData", &op::Datum::inputNetData) + .def_readwrite("outputData", &op::Datum::outputData) + .def_readwrite("cvOutputData", &op::Datum::cvOutputData) + .def_readwrite("cvOutputData3D", &op::Datum::cvOutputData3D) + .def_readwrite("poseKeypoints", &op::Datum::poseKeypoints) + .def_readwrite("poseIds", &op::Datum::poseIds) + .def_readwrite("poseScores", &op::Datum::poseScores) + .def_readwrite("poseHeatMaps", &op::Datum::poseHeatMaps) + .def_readwrite("poseCandidates", &op::Datum::poseCandidates) + .def_readwrite("faceRectangles", &op::Datum::faceRectangles) + .def_readwrite("faceKeypoints", &op::Datum::faceKeypoints) + .def_readwrite("faceHeatMaps", &op::Datum::faceHeatMaps) + .def_readwrite("handRectangles", &op::Datum::handRectangles) + .def_readwrite("handKeypoints", &op::Datum::handKeypoints) + .def_readwrite("handHeatMaps", &op::Datum::handHeatMaps) + .def_readwrite("poseKeypoints3D", &op::Datum::poseKeypoints3D) + .def_readwrite("faceKeypoints3D", &op::Datum::faceKeypoints3D) + .def_readwrite("handKeypoints3D", &op::Datum::handKeypoints3D) + .def_readwrite("cameraMatrix", &op::Datum::cameraMatrix) + .def_readwrite("cameraExtrinsics", &op::Datum::cameraExtrinsics) + .def_readwrite("cameraIntrinsics", &op::Datum::cameraIntrinsics) + .def_readwrite("scaleInputToNetInputs", &op::Datum::scaleInputToNetInputs) + .def_readwrite("netInputSizes", &op::Datum::netInputSizes) + .def_readwrite("scaleInputToOutput", &op::Datum::scaleInputToOutput) + .def_readwrite("netOutputSize", &op::Datum::netOutputSize) + .def_readwrite("scaleNetToOutput", &op::Datum::scaleNetToOutput) + .def_readwrite("elementRendered", &op::Datum::elementRendered) + ; + + // Rectangle + py::class_>(m, "Rectangle") + .def("__repr__", [](op::Rectangle &a) { return a.toString(); }) + .def(py::init<>()) + .def(py::init()) + .def_readwrite("x", &op::Rectangle::x) + .def_readwrite("y", &op::Rectangle::y) + .def_readwrite("width", &op::Rectangle::width) + .def_readwrite("height", &op::Rectangle::height) + ; + + // Point + py::class_>(m, "Point") + .def("__repr__", [](op::Point &a) { return a.toString(); }) + .def(py::init<>()) + .def(py::init()) + .def_readwrite("x", &op::Point::x) + .def_readwrite("y", &op::Point::y) + ; + + #ifdef VERSION_INFO + m.attr("__version__") = VERSION_INFO; + #else + m.attr("__version__") = "dev"; + #endif +} - typedef void* c_OP; - op::Array output; - - OP_EXPORT c_OP newOP(int logging_level, - char* output_resolution, - char* net_resolution, - char* model_pose, - float alpha_pose, - float scale_gap, - int scale_number, - float render_threshold, - int num_gpu_start, - bool disable_blending, - char* model_folder - ) { - return new OpenPose(logging_level, output_resolution, net_resolution, model_pose, alpha_pose, - scale_gap, scale_number, render_threshold, num_gpu_start, disable_blending, model_folder); - } - OP_EXPORT void delOP(c_OP op) { - delete (OpenPose *)op; - } - OP_EXPORT void forward(c_OP op, unsigned char* img, size_t rows, size_t cols, int* size, unsigned char* displayImg, bool display) { - OpenPose* openPose = (OpenPose*)op; - cv::Mat image(rows, cols, CV_8UC3, img); - cv::Mat displayImage(rows, cols, CV_8UC3, displayImg); - openPose->forward(image, output, displayImage, display); - if (output.getSize().size()) { - size[0] = output.getSize()[0]; - size[1] = output.getSize()[1]; - size[2] = output.getSize()[2]; +} + +// Numpy - op::Array interop +namespace pybind11 { namespace detail { + +template <> struct type_caster> { + public: + + PYBIND11_TYPE_CASTER(op::Array, _("numpy.ndarray")); + + // Cast numpy to op::Array + bool load(handle src, bool imp) + { + // array b(src, true); + array b = reinterpret_borrow(src); + buffer_info info = b.request(); + + if (info.format != format_descriptor::format()) + throw std::runtime_error("op::Array only supports float32 now"); + + //std::vector a(info.shape); + std::vector shape(std::begin(info.shape), std::end(info.shape)); + + // No copy + value = op::Array(shape, (float*)info.ptr); + // Copy + //value = op::Array(shape); + //memcpy(value.getPtr(), info.ptr, value.getVolume()*sizeof(float)); + + return true; } - else { - size[0] = 0; size[1] = 0; size[2] = 0; + + // Cast op::Array to numpy + static handle cast(const op::Array &m, return_value_policy, handle defval) + { + std::string format = format_descriptor::format(); + return array(buffer_info( + m.getPseudoConstPtr(),/* Pointer to buffer */ + sizeof(float), /* Size of one scalar */ + format, /* Python struct-style format descriptor */ + m.getSize().size(), /* Number of dimensions */ + m.getSize(), /* Buffer dimensions */ + m.getStride() /* Strides (in bytes) for each index */ + )).release(); } - if (display) memcpy(displayImg, displayImage.ptr(), sizeof(unsigned char)*rows*cols * 3); - } - OP_EXPORT void getOutputs(c_OP op, float* array) { - if (output.getSize().size()) - memcpy(array, output.getPtr(), output.getSize()[0] * output.getSize()[1] * output.getSize()[2] * sizeof(float)); - } - OP_EXPORT void poseFromHeatmap(c_OP op, unsigned char* img, size_t rows, size_t cols, unsigned char* displayImg, float* hm, int* size, float* ratios) { - OpenPose* openPose = (OpenPose*)op; - cv::Mat image(rows, cols, CV_8UC3, img); - cv::Mat displayImage(rows, cols, CV_8UC3, displayImg); - - std::vector>> caffeNetOutputBlob; - - for (int i = 0; i> caffeHmPtr(new caffe::Blob()); - caffeHmPtr->Reshape(1, size[1], size[2] * ((float)ratios[i] / (float)ratios[0]), size[3] * ((float)ratios[i] / (float)ratios[0])); - float* startIndex = &hm[i*size[1] * size[2] * size[3]]; - for (int d = 0; dshape()[1]; d++) { - for (int r = 0; rshape()[2]; r++) { - for (int c = 0; cshape()[3]; c++) { - int toI = d*caffeHmPtr->shape()[2] * caffeHmPtr->shape()[3] + r*caffeHmPtr->shape()[3] + c; - int fromI = d*size[2] * size[3] + r*size[3] + c; - caffeHmPtr->mutable_cpu_data()[toI] = startIndex[fromI]; - } + + }; +}} // namespace pybind11::detail + +// Numpy - cv::Mat interop +namespace pybind11 { namespace detail { + +template <> struct type_caster { + public: + + PYBIND11_TYPE_CASTER(cv::Mat, _("numpy.ndarray")); + + // Cast numpy to cv::Mat + bool load(handle src, bool) + { + /* Try a default converting into a Python */ + //array b(src, true); + array b = reinterpret_borrow(src); + buffer_info info = b.request(); + + int ndims = info.ndim; + + decltype(CV_32F) dtype; + size_t elemsize; + if (info.format == format_descriptor::format()) { + if (ndims == 3) { + dtype = CV_32FC3; + } else { + dtype = CV_32FC1; } + elemsize = sizeof(float); + } else if (info.format == format_descriptor::format()) { + if (ndims == 3) { + dtype = CV_64FC3; + } else { + dtype = CV_64FC1; + } + elemsize = sizeof(double); + } else if (info.format == format_descriptor::format()) { + if (ndims == 3) { + dtype = CV_8UC3; + } else { + dtype = CV_8UC1; + } + elemsize = sizeof(unsigned char); + } else { + throw std::logic_error("Unsupported type"); + return false; } - caffeNetOutputBlob.emplace_back(caffeHmPtr); - } - std::vector> imageSizes; - for (int i = 0; i point(cols*ratios[i], rows*ratios[i]); - imageSizes.emplace_back(point); - } + std::vector shape = {(int)info.shape[0], (int)info.shape[1]}; - openPose->poseFromHeatmap(image, caffeNetOutputBlob, output, displayImage, imageSizes); - memcpy(displayImg, displayImage.ptr(), sizeof(unsigned char)*rows*cols * 3); - // Copy back kp size - if (output.getSize().size()) { - size[0] = output.getSize()[0]; - size[1] = output.getSize()[1]; - size[2] = output.getSize()[2]; + value = cv::Mat(cv::Size(shape[1], shape[0]), dtype, info.ptr, cv::Mat::AUTO_STEP); + return true; } - else { - size[0] = 0; size[1] = 0; size[2] = 0; + + // Cast cv::Mat to numpy + static handle cast(const cv::Mat &m, return_value_policy, handle defval) + { + std::string format = format_descriptor::format(); + size_t elemsize = sizeof(unsigned char); + int dim; + switch(m.type()) { + case CV_8U: + format = format_descriptor::format(); + elemsize = sizeof(unsigned char); + dim = 2; + break; + case CV_8UC3: + format = format_descriptor::format(); + elemsize = sizeof(unsigned char); + dim = 3; + break; + case CV_32F: + format = format_descriptor::format(); + elemsize = sizeof(float); + dim = 2; + break; + case CV_64F: + format = format_descriptor::format(); + elemsize = sizeof(double); + dim = 2; + break; + default: + throw std::logic_error("Unsupported type"); + } + + std::vector bufferdim; + std::vector strides; + if (dim == 2) { + bufferdim = {(size_t) m.rows, (size_t) m.cols}; + strides = {elemsize * (size_t) m.cols, elemsize}; + } else if (dim == 3) { + bufferdim = {(size_t) m.rows, (size_t) m.cols, (size_t) 3}; + strides = {(size_t) elemsize * m.cols * 3, (size_t) elemsize * 3, (size_t) elemsize}; + } + return array(buffer_info( + m.data, /* Pointer to buffer */ + elemsize, /* Size of one scalar */ + format, /* Python struct-style format descriptor */ + dim, /* Number of dimensions */ + bufferdim, /* Buffer dimensions */ + strides /* Strides (in bytes) for each index */ + )).release(); } - } -#ifdef __cplusplus -} -#endif + }; +}} // namespace pybind11::detail #endif + diff --git a/python/openpose/openpose.py b/python/openpose/openpose.py deleted file mode 100644 index 60f43f91b..000000000 --- a/python/openpose/openpose.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -Wrap the OpenPose library with Python. -To install run `make install` and library will be stored in /usr/local/python -""" -import numpy as np -import ctypes as ct -import cv2 -import os -from sys import platform -dir_path = os.path.dirname(os.path.realpath(__file__)) - -if platform == "win32": - os.environ['PATH'] = dir_path + "/../../bin;" + os.environ['PATH'] - os.environ['PATH'] = dir_path + "/../../x64/Debug;" + os.environ['PATH'] - os.environ['PATH'] = dir_path + "/../../x64/Release;" + os.environ['PATH'] - -class OpenPose(object): - """ - Ctypes linkage - """ - if platform == "linux" or platform == "linux2": - _libop= np.ctypeslib.load_library('_openpose', dir_path+'/_openpose.so') - elif platform == "darwin": - _libop= np.ctypeslib.load_library('_openpose', dir_path+'/_openpose.dylib') - elif platform == "win32": - try: - _libop= np.ctypeslib.load_library('_openpose', dir_path+'/Release/_openpose.dll') - except OSError as e: - _libop= np.ctypeslib.load_library('_openpose', dir_path+'/Debug/_openpose.dll') - _libop.newOP.argtypes = [ - ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_char_p, ct.c_float, ct.c_float, ct.c_int, ct.c_float, ct.c_int, ct.c_bool, ct.c_char_p] - _libop.newOP.restype = ct.c_void_p - _libop.delOP.argtypes = [ct.c_void_p] - _libop.delOP.restype = None - - _libop.forward.argtypes = [ - ct.c_void_p, np.ctypeslib.ndpointer(dtype=np.uint8), - ct.c_size_t, ct.c_size_t, - np.ctypeslib.ndpointer(dtype=np.int32), np.ctypeslib.ndpointer(dtype=np.uint8), ct.c_bool] - _libop.forward.restype = None - - _libop.getOutputs.argtypes = [ - ct.c_void_p, np.ctypeslib.ndpointer(dtype=np.float32)] - _libop.getOutputs.restype = None - - _libop.poseFromHeatmap.argtypes = [ - ct.c_void_p, np.ctypeslib.ndpointer(dtype=np.uint8), - ct.c_size_t, ct.c_size_t, - np.ctypeslib.ndpointer(dtype=np.uint8), - np.ctypeslib.ndpointer(dtype=np.float32), np.ctypeslib.ndpointer(dtype=np.int32), np.ctypeslib.ndpointer(dtype=np.float32)] - _libop.poseFromHeatmap.restype = None - - def encode(self, string): - return ct.c_char_p(string.encode('utf-8')) - - def __init__(self, params): - """ - OpenPose Constructor: Prepares OpenPose object - - Parameters - ---------- - params : dict of required parameters. refer to openpose example for more details - - Returns - ------- - outs: OpenPose object - """ - self.op = self._libop.newOP(params["logging_level"], - self.encode(params["output_resolution"]), - self.encode(params["net_resolution"]), - self.encode(params["model_pose"]), - params["alpha_pose"], - params["scale_gap"], - params["scale_number"], - params["render_threshold"], - params["num_gpu_start"], - params["disable_blending"], - self.encode(params["default_model_folder"])) - - def __del__(self): - """ - OpenPose Destructor: Destroys OpenPose object - """ - self._libop.delOP(self.op) - - def forward(self, image, display = False): - """ - Forward: Takes in an image and returns the human 2D poses, along with drawn image if required - - Parameters - ---------- - image : color image of type ndarray - display : If set to true, we return both the pose and an annotated image for visualization - - Returns - ------- - array: ndarray of human 2D poses [People * BodyPart * XYConfidence] - displayImage : image for visualization - """ - shape = image.shape - displayImage = np.zeros(shape=(image.shape),dtype=np.uint8) - size = np.zeros(shape=(3),dtype=np.int32) - self._libop.forward(self.op, image, shape[0], shape[1], size, displayImage, display) - array = np.zeros(shape=(size),dtype=np.float32) - self._libop.getOutputs(self.op, array) - if display: - return array, displayImage - return array - - def poseFromHM(self, image, hm, ratios=[1]): - """ - Pose From Heatmap: Takes in an image, computed heatmaps, and require scales and computes pose - - Parameters - ---------- - image : color image of type ndarray - hm : heatmap of type ndarray with heatmaps and part affinity fields - ratios : scaling ration if needed to fuse multiple scales - - Returns - ------- - array: ndarray of human 2D poses [People * BodyPart * XYConfidence] - displayImage : image for visualization - """ - if len(ratios) != len(hm): - raise Exception("Ratio shape mismatch") - - # Find largest - hm_combine = np.zeros(shape=(len(hm), hm[0].shape[1], hm[0].shape[2], hm[0].shape[3]),dtype=np.float32) - i=0 - for h in hm: - hm_combine[i,:,0:h.shape[2],0:h.shape[3]] = h - i+=1 - hm = hm_combine - - ratios = np.array(ratios,dtype=np.float32) - - shape = image.shape - displayImage = np.zeros(shape=(image.shape),dtype=np.uint8) - size = np.zeros(shape=(4),dtype=np.int32) - size[0] = hm.shape[0] - size[1] = hm.shape[1] - size[2] = hm.shape[2] - size[3] = hm.shape[3] - - self._libop.poseFromHeatmap(self.op, image, shape[0], shape[1], displayImage, hm, size, ratios) - array = np.zeros(shape=(size[0],size[1],size[2]),dtype=np.float32) - self._libop.getOutputs(self.op, array) - return array, displayImage - - @staticmethod - def process_frames(frame, boxsize = 368, scales = [1]): - base_net_res = None - imagesForNet = [] - imagesOrig = [] - for idx, scale in enumerate(scales): - # Calculate net resolution (width, height) - if idx == 0: - net_res = (16 * int((boxsize * frame.shape[1] / float(frame.shape[0]) / 16) + 0.5), boxsize) - base_net_res = net_res - else: - net_res = (int(min(base_net_res[0], max(1, int((base_net_res[0] * scale)+0.5)/16*16))), - int(min(base_net_res[1], max(1, int((base_net_res[1] * scale)+0.5)/16*16)))) - input_res = [frame.shape[1], frame.shape[0]] - scale_factor = min((net_res[0] - 1) / float(input_res[0] - 1), (net_res[1] - 1) / float(input_res[1] - 1)) - warp_matrix = np.array([[scale_factor,0,0], - [0,scale_factor,0]]) - if scale_factor != 1: - imageForNet = cv2.warpAffine(frame, warp_matrix, net_res, flags=(cv2.INTER_AREA if scale_factor < 1. else cv2.INTER_CUBIC), borderMode=cv2.BORDER_CONSTANT, borderValue=(0,0,0)) - else: - imageForNet = frame.copy() - - imageOrig = imageForNet.copy() - imageForNet = imageForNet.astype(float) - imageForNet = imageForNet/256. - 0.5 - imageForNet = np.transpose(imageForNet, (2,0,1)) - - imagesForNet.append(imageForNet) - imagesOrig.append(imageOrig) - - return imagesForNet, imagesOrig - - @staticmethod - def draw_all(imageForNet, heatmaps, currIndex, div=4., norm=False): - netDecreaseFactor = float(imageForNet.shape[0]) / float(heatmaps.shape[2]) # 8 - resized_heatmaps = np.zeros(shape=(heatmaps.shape[0], heatmaps.shape[1], imageForNet.shape[0], imageForNet.shape[1])) - num_maps = heatmaps.shape[1] - combined = None - for i in range(0, num_maps): - heatmap = heatmaps[0,i,:,:] - resizedHeatmap = cv2.resize(heatmap, (0,0), fx=netDecreaseFactor, fy=netDecreaseFactor) - - minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(resizedHeatmap) - - if i==currIndex and currIndex >=0: - resizedHeatmap = np.abs(resizedHeatmap) - resizedHeatmap = (resizedHeatmap*255.).astype(dtype='uint8') - im_color = cv2.applyColorMap(resizedHeatmap, cv2.COLORMAP_JET) - resizedHeatmap = cv2.addWeighted(imageForNet, 1, im_color, 0.3, 0) - cv2.circle(resizedHeatmap, (int(maxLoc[0]),int(maxLoc[1])), 5, (255,0,0), -1) - return resizedHeatmap - else: - resizedHeatmap = np.abs(resizedHeatmap) - if combined is None: - combined = np.copy(resizedHeatmap); - else: - if i <= num_maps-2: - combined += resizedHeatmap; - if norm: - combined = np.maximum(0, np.minimum(1, combined)); - - if currIndex < 0: - combined /= div - combined = (combined*255.).astype(dtype='uint8') - im_color = cv2.applyColorMap(combined, cv2.COLORMAP_JET) - combined = cv2.addWeighted(imageForNet, 0.5, im_color, 0.5, 0) - cv2.circle(combined, (int(maxLoc[0]),int(maxLoc[1])), 5, (255,0,0), -1) - return combined - - -if __name__ == "__main__": - params = dict() - params["logging_level"] = 3 - params["output_resolution"] = "-1x-1" - params["net_resolution"] = "-1x368" - params["model_pose"] = "BODY_25" - params["alpha_pose"] = 0.6 - params["scale_gap"] = 0.25 - params["scale_number"] = 1 - params["render_threshold"] = 0.05 - params["num_gpu_start"] = 0 - params["disable_blending"] = False - params["default_model_folder"] = "../../../models/" - openpose = OpenPose(params) - - img = cv2.imread("../../../examples/media/COCO_val2014_000000000192.jpg") - arr, output_image = openpose.forward(img, True) - print(arr) - - while 1: - cv2.imshow("output", output_image) - cv2.waitKey(15) - diff --git a/scripts/osx/install_deps.sh b/scripts/osx/install_deps.sh index 0cd32bdeb..0202f9b2f 100755 --- a/scripts/osx/install_deps.sh +++ b/scripts/osx/install_deps.sh @@ -10,3 +10,5 @@ brew install hdf5 opencv brew install protobuf boost brew install cmake brew install viennacl +sudo pip3 install numpy +sudo pip3 install opencv-python diff --git a/scripts/travis/configure_cmake.sh b/scripts/travis/configure_cmake.sh index 321280abb..e3bd7e2d1 100755 --- a/scripts/travis/configure_cmake.sh +++ b/scripts/travis/configure_cmake.sh @@ -15,7 +15,7 @@ fi echo "WITH_PYTHON = ${WITH_PYTHON}." if [[ $WITH_PYTHON == true ]] ; then - ARGS="$ARGS -DBUILD_PYTHON=On" + ARGS="$ARGS -DBUILD_PYTHON=On -DPYTHON_EXECUTABLE=/usr/bin/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7m.so" fi # CUDA version @@ -50,3 +50,8 @@ fi echo "ARGS = ${ARGS}." cmake .. $ARGS + +# Run Cmake twice for pybind to register +if [[ $WITH_PYTHON == true ]] ; then + cmake .. $ARGS +fi \ No newline at end of file diff --git a/scripts/travis/run_tests.sh b/scripts/travis/run_tests.sh index 0c4f989c5..8c79cb630 100755 --- a/scripts/travis/run_tests.sh +++ b/scripts/travis/run_tests.sh @@ -39,7 +39,7 @@ if [[ $RUN_EXAMPLES == true ]] ; then # Python examples if [[ $WITH_PYTHON == true ]] ; then echo "Python API C++: Example 1..." - echo "TODO: Add Python examples in here..." + cd build/examples/tutorial_api_python; python openpose_python.py --net_resolution -1x32 --image_dir ../../../examples/media/ --write_json output/ --display 0 --render_pose 0 echo " " fi diff --git a/scripts/ubuntu/install_ubuntu_deps.sh b/scripts/ubuntu/install_ubuntu_deps.sh index fa4ff4464..ebcda107e 100755 --- a/scripts/ubuntu/install_ubuntu_deps.sh +++ b/scripts/ubuntu/install_ubuntu_deps.sh @@ -13,6 +13,8 @@ sudo apt-get --assume-yes install libgflags-dev libgoogle-glog-dev liblmdb-dev # Python libs sudo apt-get install python-setuptools python-dev build-essential sudo easy_install pip -sudo -H pip install --upgrade numpy protobuf +sudo -H pip install --upgrade numpy protobuf opencv-python +sudo apt-get --assume-yes install python3-pip +sudo -H pip3 install --upgrade numpy protobuf opencv-python # OpenCV 2.4 -> Added as option # sudo apt-get --assume-yes install libopencv-dev