#77 Support table partition

CHANGELOG.md

@@ -8,6 +8,7 @@ Please mark all change in change log and use the ticket from JIRA.
 
 ## Feature
 - \#12 - Pure CPU version for Milvus
+- \#77 - Support table partition
 - \#226 - Experimental shards middleware for Milvus
 
 ## Improvement

core/migration/README.md

@@ -0,0 +1,28 @@
+## Data Migration
+
+####0.3.x
+legacy data is not migrate-able for later versions
+
+####0.4.x
+legacy data can be reused directly by 0.5.x
+
+legacy data can be migrated to 0.6.x
+
+####0.5.x
+legacy data can be migrated to 0.6.x
+
+####0.6.x
+how to migrate legacy 0.4.x/0.5.x data
+
+for sqlite meta:
+```shell
+ $ sqlite3 [parth_to]/meta.sqlite < sqlite_4_to_6.sql
+```
+
+for mysql meta:
+```shell
+ $ mysql -h127.0.0.1 -uroot -p123456 -Dmilvus < mysql_4_to_6.sql
+```
+
+
+

core/migration/mysql_4_to_6.sql

@@ -0,0 +1,4 @@
+alter table Tables add column owner_table VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column partition_tag VARCHAR(255) DEFAULT '' NOT NULL;
+alter table Tables add column version VARCHAR(64) DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/migration/sqlite_4_to_6.sql

@@ -0,0 +1,4 @@
+alter table Tables add column 'owner_table' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'partition_tag' TEXT DEFAULT '' NOT NULL;
+alter table Tables add column 'version' TEXT DEFAULT '0.6.0' NOT NULL;
+update Tables set version='0.6.0';

core/src/db/DB.h

@@ -47,43 +47,68 @@ class DB {
 
     virtual Status
     CreateTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
-    DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+    DropTable(const std::string& table_id, const meta::DatesT& dates) = 0;
+
     virtual Status
     DescribeTable(meta::TableSchema& table_schema_) = 0;
+
     virtual Status
     HasTable(const std::string& table_id, bool& has_or_not_) = 0;
+
     virtual Status
     AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
+
     virtual Status
     GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
+
     virtual Status
     PreloadTable(const std::string& table_id) = 0;
+
     virtual Status
     UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
 
     virtual Status
-    InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
+    CreatePartition(const std::string& table_id, const std::string& partition_name,
+                    const std::string& partition_tag) = 0;
+
+    virtual Status
+    DropPartition(const std::string& partition_name) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          QueryResults& results) = 0;
+    DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
 
     virtual Status
-    Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
-          const meta::DatesT& dates, QueryResults& results) = 0;
+    ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
 
     virtual Status
-    Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
-          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
+    InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,
+                  IDNumbers& vector_ids_) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;
+
+    virtual Status
+    Query(const std::string& table_id, const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nq,
+          uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+          ResultDistances& result_distances) = 0;
+
+    virtual Status
+    QueryByFileID(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
+                  uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids,
+                  ResultDistances& result_distances) = 0;
 
     virtual Status
     Size(uint64_t& result) = 0;
 
     virtual Status
     CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
+
     virtual Status
     DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
+
     virtual Status
     DropIndex(const std::string& table_id) = 0;