Python state and chaining (#90)

* Simple python experiment

* Moved python lib

* Added pyfaasm submodule

* Added divide and conquer checker

* More work on python bindings

* Python benchmarks

* Strict compilation errors

* Pyfaasm

* pyfaasm

* Python input/output

* Fiddling with numpy

* numpy redis bytes

* Updated pyfaasm

* e2e python state

* Test cases for python state

* Updated pyfaasm

* Python matrix multiplication function

* More python chaining

* Test for python chaining

.gitmodules

@@ -25,3 +25,6 @@
 [submodule "third-party/llvm-project"]
 	path = third-party/llvm-project
 	url = https://github.com/llvm/llvm-project.git
+[submodule "third-party/pyfaasm"]
+	path = third-party/pyfaasm
+	url = https://github.com/Shillaker/pyfaasm.git

CMakeLists.txt

@@ -13,7 +13,7 @@ set(FAASM_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/src)
 set(FAASM_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/include)
 set(FAASM_LIB_C_DIR ${CMAKE_CURRENT_LIST_DIR}/lib-c)
 set(FAASM_LIB_CPP_DIR ${CMAKE_CURRENT_LIST_DIR}/lib-cpp)
-set(FAASM_PYTHON_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/python)
+set(FAASM_PYTHON_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/lib-pyinit)
 set(FAASM_WAVM_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/third-party/WAVM/Lib)
 
 option(FAASM_STATIC_LIBS "Statically link Faasm libs" ON)
@@ -38,7 +38,7 @@ else()
 endif()
 
 # Switch on WAVM stack traces in debug (potential performance gain?)
-if(CMAKE_BUILD_TYPE MATCHES DEBUG)
+if(CMAKE_BUILD_TYPE MATCHES Debug)
     set(WAVM_ENABLE_UNWIND ON CACHE BOOL "WAVM unwind")
 else()
     set(WAVM_ENABLE_UNWIND OFF CACHE BOOL "WAVM unwind")
@@ -101,7 +101,7 @@ if (FAASM_BUILD_TYPE STREQUAL "knative-native")
 
     add_subdirectory(func)
     add_subdirectory(lib-cpp)
-    add_subdirectory(python)
+    add_subdirectory(lib-pyinit)
 
     add_subdirectory(src/emulator)
     add_subdirectory(src/endpoint)
@@ -208,7 +208,7 @@ else ()
 
     # Faasm libraries
     add_subdirectory(lib-cpp)
-    add_subdirectory(python)
+    add_subdirectory(lib-pyinit)
 
     # Custom malloc
     add_subdirectory(third-party/malloc)

bin/spawn_faasm.sh

@@ -11,7 +11,6 @@ fi
 export CGROUP_MODE=off
 export NETNS_MODE=off
 export LOG_LEVEL=off 
-export FS_MODE=on
 
 spawn_script="${HOME}/faasm/bench/bin/bench_mem sleep_long"
 

bin/worker-entrypoint.sh

@@ -5,7 +5,7 @@ set -e
 pushd /faasm/build >> /dev/null
 
 # Run codegen for some functions
-ORIGINAL_STORAGE=$FUNCTION_STORAGE
+ORIGINAL_FUNCTION_STORAGE=$FUNCTION_STORAGE
 export FUNCTION_STORAGE=local
 echo "Temporarily changing function storage from ${ORIGINAL_FUNCTION_STORAGE} to local"
 

docker-compose-ibm.env

@@ -2,7 +2,6 @@
 LOG_LEVEL=debug
 CGROUP_MODE=off
 NETNS_MODE=off
-FS_MODE=on
 IBM_API_KEY=
 
 # Redis

docker-compose.env

@@ -4,7 +4,6 @@ FILESERVER_URL=http://upload:8002
 LOG_LEVEL=debug
 CGROUP_MODE=on
 NETNS_MODE=off
-FS_MODE=on
 
 # Redis
 REDIS_STATE_HOST=redis-state

docker-compose.yml

@@ -27,28 +27,48 @@ services:
     depends_on:
       - redis-queue
       - redis-state
-    env_file:
-      - docker-compose.env
     ports:
       - "8081:8080"
     privileged: yes
     networks:
       faasm-knative:
         ipv4_address: 172.28.1.2
+    environment:
+      - FUNCTION_STORAGE=fileserver
+      - FILESERVER_URL=http://upload:8002
+      - LOG_LEVEL=debug
+      - CGROUP_MODE=on
+      - NETNS_MODE=off
+      - REDIS_STATE_HOST=redis-state
+      - REDIS_QUEUE_HOST=redis-queue
+      - THREADS_PER_WORKER=5
+      - MAX_IN_FLIGHT_RATIO=1
+      - MAX_WORKERS_PER_FUNCTION=4
+      - GLOBAL_MESSAGE_TIMEOUT=600000
 
   worker-2:
     image: faasm/knative-worker:latest
     depends_on:
       - redis-queue
       - redis-state
-    env_file:
-      - docker-compose.env
     ports:
       - "8082:8080"
     privileged: yes
     networks:
       faasm-knative:
         ipv4_address: 172.28.1.3
+    environment:
+      - FUNCTION_STORAGE=fileserver
+      - FILESERVER_URL=http://upload:8002
+      - LOG_LEVEL=debug
+      - CGROUP_MODE=on
+      - NETNS_MODE=off
+      - REDIS_STATE_HOST=redis-state
+      - REDIS_QUEUE_HOST=redis-queue
+      - THREADS_PER_WORKER=5
+      - MAX_IN_FLIGHT_RATIO=1
+      - MAX_WORKERS_PER_FUNCTION=4
+      - GLOBAL_MESSAGE_TIMEOUT=600000
 
   proxy:
     build:
@@ -70,10 +90,13 @@ services:
     depends_on:
       - redis-queue
       - redis-state
-    env_file:
-      - docker-compose.env
     networks:
       - faasm-knative
     volumes:
       - /usr/local/faasm/:/usr/local/faasm/
       - /usr/local/code/faasm/wasm/:/usr/local/code/faasm/wasm/
+    environment:
+      - FUNCTION_STORAGE=local
+      - LOG_LEVEL=debug
+      - REDIS_STATE_HOST=redis-state
+      - REDIS_QUEUE_HOST=redis-queue

docker/knative-native-base.dockerfile

@@ -3,6 +3,9 @@ FROM faasm/base
 # Install python lib for python functions
 RUN apt-get install libpython3-dev
 
+# Install pyfaasm for interacting with emulator
+RUN pip install pyfaasm
+
 COPY . /usr/local/code/faasm
 
 # Nuke and recreate the build dir

docs/benchmarks.md

@@ -76,19 +76,6 @@ inv bench-mem
 
 Results are written to `/root/faasm/results/runtime-bench-mem.csv` (assuming your root home is `/root`).
 
-### Other runtimes
-
-It would be nice to benchmark these similar overheads against Firecracker, however I haven't been able to script it
-up yet. It can be set up by running:
-
-```
-cd ansible
-ansible-playbook firecracker.yml --ask-become-pass
-```
-
-Lucet would also be interesting but it doesn't currently provide a
-[full isolation environment](https://github.com/fastly/lucet/security/policy), therefore is not really comparable.
-
 ## Capacity
 
 Capacity measurements aim to work out the maximum number of concurrent workers we can sustain on a given box for both Faasm and Docker.

func/python/CMakeLists.txt

@@ -19,9 +19,9 @@ include_directories(${PYTHON_INCLUDE_DIRS})
 faasm_dynamic_func(py_func py_func.cpp)
 
 if (FAASM_BUILD_TYPE STREQUAL "wasm")
-    target_link_libraries(py_func pyfaasm ${PYTHON_LIBRARIES})
+    target_link_libraries(py_func pyinit ${PYTHON_LIBRARIES})
 elseif (FAASM_BUILD_TYPE STREQUAL "knative-native")
-    target_link_libraries(py_func-knative pyfaasm ${PYTHON_LIBRARIES} util)
+    target_link_libraries(py_func-knative pyinit ${PYTHON_LIBRARIES} util)
 else()
-    target_link_libraries(py_func pyfaasm ${PYTHON_LIBRARIES} util)
+    target_link_libraries(py_func pyinit ${PYTHON_LIBRARIES} util)
 endif()

func/python/chain.py

@@ -0,0 +1,46 @@
+from pyfaasm.core import getFunctionIdx, getInput, chainThisWithInput, registerFunction, IS_NATIVE_PYTHON, awaitCall
+
+idx = getFunctionIdx()
+print("Got function index {}".format(idx))
+
+
+def chainOne(input_bytes):
+    expected = b'1234'
+    print("Chained 1: {} {}".format(input_bytes, expected))
+    if input_bytes != expected:
+        exit(1)
+
+
+def chainTwo(input_bytes):
+    expected = b'5678'
+    print("Chained 2: {} {}".format(input_bytes, expected))
+    if input_bytes != expected:
+        exit(1)
+
+
+if IS_NATIVE_PYTHON:
+    print("Running native python")
+    registerFunction(1, chainOne)
+    registerFunction(2, chainTwo)
+else:
+    print("Not running native python")
+
+if idx == 0:
+    print("Main chaining entry point")
+    call_a = chainThisWithInput(1, b'1234')
+    call_b = chainThisWithInput(2, b'5678')
+
+    print("Awaiting calls {} and {}".format(call_a, call_b))
+
+    res_a = awaitCall(call_a)
+    res_b = awaitCall(call_b)
+
+    if res_a != 0 or res_b != 0:
+        print("Chained functions failed: {} {}".format(res_a, res_b))
+        exit(1)
+
+elif idx == 1:
+    chainOne(getInput())
+
+elif idx == 2:
+    chainTwo(getInput())

func/python/echo.py

@@ -0,0 +1,7 @@
+from pyfaasm.core import getInput, setOutput
+
+i = getInput()
+
+print("Got input {}".format(i))
+
+setOutput(bytes(i))

func/python/mat_mul.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+from pyfaasm.core import getFunctionIdx, getInput
+from pyfaasm.matrix import divide_and_conquer, distributed_divide_and_conquer, MATRIX_SIZE
+
+# This is the Faasm entry point. If we've been chained,
+# we need to invoke the worker, otherwise run the main function
+print("Getting function index for matrix multiplication")
+idx = getFunctionIdx()
+
+if idx == 0:
+    print("Setting up matrix multiplication problem")
+    # Set up the problem
+    mat_a = np.random.rand(MATRIX_SIZE, MATRIX_SIZE)
+    mat_b = np.random.rand(MATRIX_SIZE, MATRIX_SIZE)
+
+    divide_and_conquer()
+else:
+    # Run the worker
+    print("Running matrix multiplication worker")
+    input_bytes = getInput()
+    print("Got input bytes: {}".format(input_bytes))
+    distributed_divide_and_conquer(input_bytes)
+

func/python/py_func.cpp

@@ -1,5 +1,5 @@
 #include <faasm/faasm.h>
-#include <faasm/pyfaasm.h>
+#include <faasm/pyinit.h>
 
 #include <Python.h>
 
@@ -8,9 +8,13 @@
 
 FAASM_ZYGOTE() {
     setUpPyEnvironment();
-    setUpPyNumpy();
     Py_InitializeEx(0);
 
+    setUpPyNumpy();
+
+    // Import numpy up front
+    // PyImport_ImportModule("numpy");
+
     printf("\n\nPython initialised\n");
 
     return 0;
@@ -22,6 +26,8 @@ FAASM_MAIN_FUNC() {
     char *user = faasmGetPythonUser();
     char *funcName = faasmGetPythonFunc();
 
+    printf("Running Python function %s/%s\n", user, funcName);
+
     auto filePath = new char[50 + strlen(funcName)];
 
 #ifdef __wasm__

func/python/state_test_read.py

@@ -0,0 +1,29 @@
+from pyfaasm.core import getState, getStateOffset
+
+key = "pyStateTest"
+valueLen = 10
+expected = b'0199956789'
+
+# Check the full value
+actual = getState(key, valueLen)
+if actual != expected:
+    msg = "Mismatch: actual {}, expected {})".format(actual, expected)
+    print(msg)
+    exit(1)
+
+# Check a couple of segments
+actualSegmentA = getStateOffset(key, valueLen, 0, 3)
+expectedA = b'019'
+if actualSegmentA != expectedA:
+    msg = "Mismatched segment: actual {}, expected {})".format(actualSegmentA, expectedA)
+    print(msg)
+    exit(1)
+
+actualSegmentB = getStateOffset(key, valueLen, 4, 4)
+expectedB = b'9567'
+if actualSegmentB != expectedB:
+    msg = "Mismatched segment: actual {}, expected {})".format(actualSegmentB, expectedB)
+    print(msg)
+    exit(1)
+
+print("Successful Python state reading check")