Feature/oram

.gitignore

@@ -40,7 +40,10 @@ bin
 *.env
 generate
 
-test/integrate
+test/integrate/person_list
+test/integrate/project.py
 test/project_path.h
+test/integrate/__pycache__/
+test/integrate/js/
 .vscode/
-hash_hex/
+hash_hex/

CMakeLists.txt

@@ -4,7 +4,7 @@
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 cmake_minimum_required(VERSION 3.12)
-project(YPC VERSION 0.5.1)
+project(YPC VERSION 0.5.2)
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
 

build.sh

@@ -38,6 +38,7 @@ create_signed_so() {
                            ["person_first_match"]="example/personlist/first_match/enclave/enclave.config.xml"
                            ["person_first_match_multi"]="example/multi_personlist/first_match/enclave/enclave.config.xml"
                            ["person_first_match_multi_offchain"]="example/multi_personlist/first_match/enclave_for_offchain/enclave.config.xml"
+                           ["convert_sealed_file"]="example/convert/enclave/enclave.config.xml"
                           )
 
   for filename in ${!enclave_libs[@]}; do

example/convert/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(enclave)
+add_subdirectory(enclave2)

example/convert/common.h

@@ -0,0 +1,7 @@
+#pragma once
+#include "common_t.h"
+#include "ypc/core/blockfile.h"
+#include "ypc/core/byte.h"
+#include "ypc/corecommon/package.h"
+typedef ypc::blockfile<0x82, 1024 * 1024, 256 * 64 * 1024> file_t;
+

example/convert/common_t.h

@@ -0,0 +1,36 @@
+#pragma once
+#include <ff/net/middleware/ntpackage.h>
+#include <ff/util/ntobject.h>
+
+define_nt(RYXXBZ, std::string, "RYXXBZ");
+define_nt(XM, std::string, "XM");
+define_nt(CYM, std::string, "CYM");
+define_nt(XBDM, std::string, "XBDM");
+define_nt(FWXXBZ, std::string, "FWXXBZ");
+define_nt(XP, std::string, "XP");
+define_nt(DWMC, std::string, "DWMC");
+define_nt(ZJHM, std::string, "ZJHM");
+define_nt(GJDM, std::string, "GJDM");
+define_nt(MZDM, std::string, "MZDM");
+define_nt(JGSSXDM, std::string, "JGSSXDM");
+define_nt(HKXZFLYDM, std::string, "HKXZFLYDM");
+define_nt(HLXDM, std::string, "HLXDM");
+define_nt(HJDZ_XZQHDM, std::string, "HJDZXZQHDM");
+define_nt(SJJZD_XZQHDM, std::string, "SJJZDXZQHDM");
+define_nt(SJJZD_QHNXXDZ, std::string, "SJJZDQHNXXDZ");
+define_nt(XLDM, std::string, "XLDM");
+define_nt(TSSFDM, std::string, "TSSFDM");
+define_nt(CSQR, std::string, "CSQR");
+define_nt(LXDH, std::string, "LXDH");
+define_nt(HYZKDM, std::string, "HYZKDM");
+define_nt(DJR_XM, std::string, "DJR_XM");
+define_nt(DJR_GMSFZHM, std::string, "DJRGMSFZHM");
+define_nt(DJR_LXDH, std::string, "DJRLXDH");
+define_nt(GXSJ, std::string, "GXSJ");
+define_nt(SJZT, std::string, "SJZT");
+
+typedef ff::util::ntobject<
+    RYXXBZ, XM, CYM, XBDM, FWXXBZ, XP, DWMC, ZJHM, GJDM, MZDM, JGSSXDM,
+    HKXZFLYDM, HLXDM, HJDZ_XZQHDM, SJJZD_XZQHDM, SJJZD_QHNXXDZ, XLDM, TSSFDM,
+    CSQR, LXDH, HYZKDM, DJR_XM, DJR_GMSFZHM, DJR_LXDH, GXSJ, SJZT>
+    row_t;

example/convert/enclave/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(T_SRCS eparser.cpp)
+
+add_ypc_applet(convert_sealed_file
+  CRYPTO stdeth
+  SRCS ${T_SRCS})
+
+if(SGX_MODE STREQUAL "Debug")
+  enclave_sign(convert_sealed_file KEY enclave_private.pem
+    CONFIG enclave.config.debug.xml)
+else()
+  enclave_sign(convert_sealed_file KEY enclave_private.pem
+    CONFIG enclave.config.xml)
+endif()

example/convert/enclave/convert_parser.h

@@ -0,0 +1,193 @@
+#include "ypc/corecommon/package.h"
+#include "ypc/stbox/ebyte.h"
+#include "ypc/stbox/stx_common.h"
+#ifdef EXAMPLE_FM_NORMAL
+#include <glog/logging.h>
+typedef ypc::bytes bytes;
+#else
+#include "ypc/core_t/analyzer/data_source.h"
+#include "ypc/stbox/tsgx/log.h"
+typedef stbox::bytes bytes;
+#endif
+#include "user_type.h"
+#include "ypc/corecommon/data_source.h"
+#include "ypc/corecommon/to_type.h"
+#include <hpda/extractor/raw_data.h>
+#include <hpda/output/memory_output.h>
+#include <hpda/processor/query/filter.h>
+#include <string.h>
+
+
+#include "ypc/core_t/analyzer/eparser_t_interface.h"
+#include "ypc/common/crypto_prefix.h"
+#include "ypc/corecommon/crypto/gmssl.h"
+#include "ypc/corecommon/crypto/stdeth.h"
+#include "ypc/corecommon/oram_types.h"
+#include <random>
+#include <hpda/processor/transform/split.h>
+
+
+using oram_ntt = ypc::oram::nt<bytes>;
+using ntt = ypc::nt<bytes>;
+
+
+
+
+
+define_nt(input_buf, std::string);
+typedef ff::net::ntpackage<0, input_buf> input_buf_t;
+
+
+class crypto_base {
+public:
+  virtual uint32_t encrypt_message_with_prefix(const bytes &public_key,
+                                               const bytes &data,
+                                               uint32_t prefix,
+                                               bytes &cipher) = 0;
+  virtual uint32_t hash_256(const bytes &msg, bytes &hash) = 0;
+};
+using crypto_ptr_t = std::shared_ptr<crypto_base>;
+template <typename Crypto> class crypto_tool : public crypto_base {
+public:
+  using crypto_t = Crypto;
+  virtual uint32_t encrypt_message_with_prefix(const bytes &public_key,
+                                               const bytes &data,
+                                               uint32_t prefix,
+                                               bytes &cipher) {
+    return crypto_t::encrypt_message_with_prefix(public_key, data, prefix,
+                                                 cipher);
+  }
+  virtual uint32_t hash_256(const bytes &msg, bytes &hash) {
+    return crypto_t::hash_256(msg, hash);
+  }
+};
+
+
+class convert_parser {
+public:
+  convert_parser() {}
+  convert_parser(ypc::data_source<bytes> *source) : m_source(source){};
+
+  inline bytes do_parse(const bytes &param) {
+    bytes result;
+
+    LOG(INFO) << "do convert_parse";
+    ypc::to_type<bytes, user_item_t> converter(m_source);
+
+    crypto_ptr_t crypto_ptr = std::make_shared<crypto_tool<ypc::crypto::eth_sgx_crypto>>();
+    bytes pub_key(param);
+
+
+
+    LOG(INFO) << "create id_map";
+
+    size_t batch_size = 0;
+    size_t item_size = 0;
+    uint64_t batch_num = 0; // the number of batch 
+    uint64_t full_num = 0;
+    uint64_t last_num = 0;
+    uint64_t item_num = 0;
+
+    std::vector<oram_ntt::id_map_pair> id_map_array;
+    uint32_t batch_id = 1;
+
+    hpda::processor::internal::filter_impl<user_item_t> match2(
+        &converter, [&](const user_item_t &v) {
+          if(item_size == 0) {
+            typename ypc::cast_obj_to_package<user_item_t>::type pt = v;
+            auto item_data = ypc::make_bytes<bytes>::for_package(pt);
+            item_size = item_data.size();
+          }
+
+          std::string item_index_field = v.get<ZJHM>();
+
+          input_buf_t item_index_field_pkg;
+          item_index_field_pkg.set<input_buf>(item_index_field);
+          bytes item_index_field_bytes = ypc::make_bytes<bytes>::for_package(item_index_field_pkg);
+          bytes item_index_field_hash;
+          crypto_ptr->hash_256(item_index_field_bytes, item_index_field_hash);
+
+          std::shared_ptr<oram_ntt::id_map_pair> k_v(new oram_ntt::id_map_pair());
+          k_v->set<oram_ntt::item_index_field_hash, oram_ntt::block_id>(item_index_field_hash, batch_id);
+          id_map_array.push_back(*k_v);          
+
+          ++item_num;
+
+          batch_size += item_size;
+          if (batch_size >= ypc::utc::max_item_size) {
+
+            if(full_num == 0) {
+              full_num = item_num;
+            }
+
+            item_num = 0;
+            batch_size = 0;
+
+            ++batch_id;
+            ++batch_num;
+          }
+
+          return false;
+        });
+    match2.get_engine()->run();
+
+    if(item_num > 0) {
+      last_num = item_num;
+      ++batch_num;
+    }
+
+    oram_ntt::id_map_t id_map_pkg;
+    id_map_pkg.set<oram_ntt::id_map>(id_map_array);
+    id_map_array.clear();
+    bytes id_map_bytes = ypc::make_bytes<bytes>::for_package(id_map_pkg);
+
+
+    LOG(INFO) << "write header";
+
+    ypc::oram::header osf_header{};
+    osf_header.block_num = batch_num;
+    uint32_t real_bucket_num = ceil(static_cast<double>(osf_header.block_num) / ypc::oram::BucketSizeZ);
+    osf_header.level_num_L = ceil(log2(real_bucket_num + 1)) - 1; 
+    osf_header.bucket_num_N = (1 << (osf_header.level_num_L + 1)) - 1;
+    osf_header.id_map_filepos = sizeof(osf_header);
+    osf_header.oram_tree_filepos = osf_header.id_map_filepos + id_map_bytes.size();
+
+    osf_header.item_size = item_size;
+    if(full_num > 0) {
+      osf_header.item_num_each_batch = full_num;
+    } else {
+      osf_header.item_num_each_batch = last_num;
+    }
+
+    auto ret = stbox::ocall_cast<uint32_t>(write_convert_data_structure)
+      (0, (uint8_t *)&osf_header, sizeof(osf_header));
+    if (ret != stbox::stx_status::success) {
+      LOG(ERROR) << "write_convert_data_structure ocall fail!";
+      return result;
+    }
+
+    LOG(INFO) << "write id map";
+
+    int32_t id_map_size = 8000000;
+    for(int i = 0; i <= id_map_bytes.size(); i += id_map_size) {
+      if(i + id_map_size <= id_map_bytes.size()) {
+        ret = stbox::ocall_cast<uint32_t>(write_convert_data_structure)
+          (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_size);
+      } else {
+        ret = stbox::ocall_cast<uint32_t>(write_convert_data_structure)
+          (osf_header.id_map_filepos + i, id_map_bytes.data() + i, id_map_bytes.size() - i);
+      }
+
+      if (ret != stbox::stx_status::success) {
+        LOG(ERROR) << "write_convert_data_structure ocall fail!";
+        return result;
+      }
+    }
+
+
+    return result;
+  }
+
+protected:
+  ypc::data_source<bytes> *m_source;
+};

example/convert/enclave/enclave.config.debug.xml

@@ -0,0 +1,12 @@
+<EnclaveConfiguration>
+  <ProdID>0</ProdID>
+  <ISVSVN>0</ISVSVN>
+  <StackMaxSize>0x50000</StackMaxSize>
+  <HeapMaxSize>0x80000000</HeapMaxSize>
+  <TCSNum>10</TCSNum>
+  <TCSPolicy>1</TCSPolicy>
+  <!-- Recommend changing 'DisableDebug' to 1 to make the enclave undebuggable for enclave release -->
+  <DisableDebug>0</DisableDebug>
+  <MiscSelect>0</MiscSelect>
+  <MiscMask>0xFFFFFFFF</MiscMask>
+</EnclaveConfiguration>

example/convert/enclave/enclave.config.xml

@@ -0,0 +1,12 @@
+<EnclaveConfiguration>
+  <ProdID>0</ProdID>
+  <ISVSVN>0</ISVSVN>
+  <StackMaxSize>0x50000</StackMaxSize>
+  <HeapMaxSize>0x80000000</HeapMaxSize>
+  <TCSNum>10</TCSNum>
+  <TCSPolicy>1</TCSPolicy>
+  <!-- Recommend changing 'DisableDebug' to 1 to make the enclave undebuggable for enclave release -->
+  <DisableDebug>1</DisableDebug>
+  <MiscSelect>0</MiscSelect>
+  <MiscMask>0xFFFFFFFF</MiscMask>
+</EnclaveConfiguration>

example/convert/enclave/enclave.lds

@@ -0,0 +1,11 @@
+enclave.so
+{
+    global:
+        g_global_data_sim;
+        g_global_data;
+        enclave_entry;
+        g_peak_heap_used;
+        g_peak_rsrv_mem_committed;
+    local:
+        *;
+};

example/convert/enclave/enclave_debug.lds

@@ -0,0 +1,11 @@
+enclave.so
+{
+    global:
+        g_global_data_sim;
+        g_global_data;
+        enclave_entry;
+        g_peak_heap_used;
+        g_peak_rsrv_mem_committed;
+    local:
+        *;
+};

example/convert/enclave/enclave_private.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG4gIBAAKCAYEAroOogvsj/fZDZY8XFdkl6dJmky0lRvnWMmpeH41Bla6U1qLZ
+AmZuyIF+mQC/cgojIsrBMzBxb1kKqzATF4+XwPwgKz7fmiddmHyYz2WDJfAjIveJ
+ZjdMjM4+EytGlkkJ52T8V8ds0/L2qKexJ+NBLxkeQLfV8n1mIk7zX7jguwbCG1Pr
+nEMdJ3Sew20vnje+RsngAzdPChoJpVsWi/K7cettX/tbnre1DL02GXc5qJoQYk7b
+3zkmhz31TgFrd9VVtmUGyFXAysuSAb3EN+5VnHGr0xKkeg8utErea2FNtNIgua8H
+ONfm9Eiyaav1SVKzPHlyqLtcdxH3I8Wg7yqMsaprZ1n5A1v/levxnL8+It02KseD
+5HqV4rf/cImSlCt3lpRg8U5E1pyFQ2IVEC/XTDMiI3c+AR+w2jSRB3Bwn9zJtFlW
+KHG3m1xGI4ck+Lci1JvWWLXQagQSPtZTsubxTQNx1gsgZhgv1JHVZMdbVlAbbRMC
+1nSuJNl7KPAS/VfzAgEDAoIBgHRXxaynbVP5gkO0ug6Qw/E27wzIw4SmjsxG6Wpe
+K7kfDeRskKxESdsA/xCrKkwGwhcx1iIgS5+Qscd1Yg+1D9X9asd/P7waPmWoZd+Z
+AhlKwhdPsO7PiF3e1AzHhGQwsUTt/Y/aSI1MpHBvy2/s1h9mFCslOUxTmWw0oj/Q
+ldIEgWeNR72CE2+jFIJIyml6ftnb6qzPiga8Bm48ubKh0kvySOqnkmnPzgh+JBD6
+JnBmtZbfPT97bwTT+N6rnPqOOApvfHPf15kWI8yDbprG1l4OCUaIUH1AszxLd826
+5IPM+8gINLRDP1MA6azECPjTyHXhtnSIBZCyWSVkc05vYmNXYUNiXWMajcxW9M02
+wKzFELO8NCEAkaTPxwo4SCyIjUxiK1LbQ9h8PSy4c1+gGP4LAMR8xqP4QKg6zdu9
+osUGG/xRe/uufgTBFkcjqBHtK5L5VI0jeNIUAgW/6iNbYXjBMJ0GfauLs+g1VsOm
+WfdgXzsb9DYdMa0OXXHypmV4GwKBwQDUwQj8RKJ6c8cT4vcWCoJvJF00+RFL+P3i
+Gx2DLERxRrDa8AVGfqaCjsR+3vLgG8V/py+z+dxZYSqeB80Qeo6PDITcRKoeAYh9
+xlT3LJOS+k1cJcEmlbbO2IjLkTmzSwa80fWexKu8/Xv6vv15gpqYl1ngYoqJM3pd
+vzmTIOi7MKSZ0WmEQavrZj8zK4endE3v0eAEeQ55j1GImbypSf7Idh7wOXtjZ7WD
+Dg6yWDrri+AP/L3gClMj8wsAxMV4ZR8CgcEA0fzDHkFa6raVOxWnObmRoDhAtE0a
+cjUj976NM5yyfdf2MrKy4/RhdTiPZ6b08/lBC/+xRfV3xKVGzacm6QjqjZrUpgHC
+0LKiZaMtccCJjLtPwQd0jGQEnKfMFaPsnhOc5y8qVkCzVOSthY5qhz0XNotHHFmJ
+gffVgB0iqrMTvSL7IA2yqqpOqNRlhaYhNl8TiFP3gIeMtVa9rZy31JPgT2uJ+kfo
+gV7sdTPEjPWZd7OshGxWpT6QfVDj/T9T7L6tAoHBAI3WBf2DFvxNL2KXT2QHAZ9t
+k3imC4f7U+wSE6zILaDZyzygA4RUbwG0gv8/TJVn2P/Eynf76DuWHGlaiLWnCbSz
+Az2DHBQBBaku409zDQym3j1ugMRjzzSQWzJg0SIyBH3hTmnYcn3+Uqcp/lEBvGW6
+O+rsXFt3pukqJmIV8HzLGGaLm62BHUeZf3dyWm+i3p/hQAL7Xvu04QW70xuGqdr5
+afV7p5eaeQIJXyGQJ0eylV/90+qxjMKiB1XYg6WYvwKBwQCL/ddpgOdHJGN8uRom
+e7Zq0Csi3hGheMKlKbN3vcxT5U7MdyHtTZZOJbTvxKNNUNYH/8uD+PqDGNneb29G
+BfGzvI3EASyLIcGZF3OhKwZd0jUrWk2y7Vhob91jwp2+t73vdMbkKyI4mHOuXvGv
+fg95si9oO7EBT+Oqvhccd2J+F1IVXncccYnF4u5ZGWt5lLewN/pVr7MjjykeaHqN
+t+rfnQam2psA6fL4zS2zTmZPzR2tnY8Y1GBTi0Ko1OKd1HMCgcAb5cB/7/AQlhP9
+yQa04PLH9ygQkKKptZp7dy5WcWRx0K/hAHRoi2aw1wZqfm7VBNu2SLcs90kCCCxp
+6C5sfJi6b8NpNbIPC+sc9wsFr7pGo9SFzQ78UlcWYK2Gu2FxlMjonhka5hvo4zvg
+WxlpXKEkaFt3gLd92m/dMqBrHfafH7VwOJY2zT3WIpjwuk0ZzmRg5p0pG/svVQEH
+NZmwRwlopysbR69B/n1nefJ84UO50fLh5s5Zr3gBRwbWNZyzhXk=
+-----END RSA PRIVATE KEY-----

example/convert/enclave/eparser.cpp

@@ -0,0 +1,13 @@
+#include "ypc/core_t/analyzer/algo_wrapper.h"
+#include "ypc/core_t/analyzer/macro.h"
+#include "ypc/corecommon/crypto/stdeth.h"
+#include "ypc/corecommon/crypto/gmssl.h"
+
+#include "convert_parser.h"
+
+ypc::algo_wrapper<ypc::crypto::eth_sgx_crypto, ypc::convert_sealed_data_stream,
+                  convert_parser,
+                  ypc::onchain_result<ypc::crypto::eth_sgx_crypto>>
+    pw;
+
+YPC_PARSER_IMPL(pw);