Merge pull request milvus-io#20 from XuanYang-cn/master

[ISSUE-milvus-io#19]

CHANGELOG.md

@@ -16,16 +16,19 @@
 - \#6 ISSUE   
     - Status quick-check-success
     - Log message more understandable
-    - Status code related to Thrift Exception 
-    - Operations before connect will raise NotConnectError, Adding UNKNOWN Status 
-     
+    - Status code related to Thrift Exception
+    - Operations before connect will raise NotConnectError, Adding UNKNOWN Status
+
 - \#8 ISSUE
     - Add new api: search_vectors_by_file
     - fix some bugs
-    
+
 - \#17 ISSUE
     - Implement has_table interface
     - fix spelling error, reformat as PEP8
+
+- \#19 ISSUE
+    - Hide Prepare object and support old version
 ### Task
 
 - \#1 Build Repository

README.md

@@ -46,7 +46,7 @@ Add a new `table`
 
 First using `Prepare` to create param
 ```python
->>> param = Prepare.table_schema(table_name='test01', dimension=256, index_type=IndexType.IDMAP,
+>>> param = Prepare.table_schema(table_name='test01', dimension=256, index_type=IndexType.FLAT,
                                     store_raw_vector=False)
 ```
 Then create `table`

TODO

@@ -2,5 +2,7 @@
 a. Support numpy in add_vector
 b. Support outside config for transport and protocol
 c. LOG Config
-d. Hide Prepare object
+d. Hide Prepare object  >> CHECK 20190624
 e. Parse Range to time: (year-1900)*1000 + (month-1)*100 + day, [start, end)
+f. create_table param
+g. search_in_files file_ids support int, using str(ids)

examples/new_example.py

@@ -0,0 +1,91 @@
+# This program demos how to connect to Milvus vector database,
+# create a vector table,
+# insert 10 vectors,
+# and execute a vector similarity search.
+
+from milvus import Milvus, IndexType, Status
+import time
+
+# Milvus server IP address and port.
+# You may need to change _HOST and _PORT accordingly.
+_HOST = '127.0.0.1'
+_PORT = '19530'
+
+
+def main():
+    milvus = Milvus()
+
+    # Connect to Milvus server
+    # You may need to change _HOST and _PORT accordingly
+    param = {'host': _HOST, 'port': _PORT}
+    status = milvus.connect(**param)
+
+    # Create table demo_table if it dosen't exist.
+    table_name = 'demo_table'
+
+    if not milvus.has_table(table_name):
+        param = {
+            'table_name': table_name,
+            'dimension': 16,
+            'index_type': IndexType.FLAT,
+            'store_raw_vector': False
+        }
+
+        milvus.create_table(param)
+
+    # Show tables in Milvus server
+    _, tables = milvus.show_tables()
+
+    # Describe demo_table
+    _, table = milvus.describe_table(table_name)
+
+    # create 10 vectors with 16 dimension
+    vectors = [
+        [0.66, 0.01, 0.29, 0.64, 0.75, 0.94, 0.26, 0.79, 0.61, 0.11, 0.25, 0.50, 0.74, 0.37, 0.28, 0.63],
+        [0.77, 0.65, 0.57, 0.68, 0.29, 0.93, 0.17, 0.15, 0.95, 0.09, 0.78, 0.37, 0.76, 0.21, 0.42, 0.15],
+        [0.61, 0.38, 0.32, 0.39, 0.54, 0.93, 0.09, 0.81, 0.52, 0.30, 0.20, 0.59, 0.15, 0.27, 0.04, 0.37],
+        [0.33, 0.03, 0.87, 0.47, 0.79, 0.61, 0.46, 0.77, 0.62, 0.70, 0.85, 0.01, 0.30, 0.41, 0.74, 0.98],
+        [0.19, 0.80, 0.03, 0.75, 0.22, 0.49, 0.52, 0.91, 0.40, 0.91, 0.79, 0.08, 0.27, 0.16, 0.07, 0.24],
+        [0.44, 0.36, 0.16, 0.88, 0.30, 0.79, 0.45, 0.31, 0.45, 0.99, 0.15, 0.93, 0.37, 0.25, 0.78, 0.84],
+        [0.33, 0.37, 0.59, 0.66, 0.76, 0.11, 0.19, 0.38, 0.14, 0.37, 0.97, 0.50, 0.08, 0.69, 0.16, 0.67],
+        [0.68, 0.97, 0.20, 0.13, 0.30, 0.16, 0.85, 0.21, 0.26, 0.17, 0.81, 0.96, 0.18, 0.40, 0.13, 0.74],
+        [0.11, 0.26, 0.44, 0.91, 0.89, 0.79, 0.98, 0.91, 0.09, 0.45, 0.07, 0.88, 0.71, 0.35, 0.97, 0.41],
+        [0.17, 0.54, 0.61, 0.58, 0.25, 0.63, 0.65, 0.71, 0.26, 0.80, 0.28, 0.77, 0.69, 0.02, 0.63, 0.60],
+    ]
+    # Insert vectors into demo_table
+    status, ids = milvus.add_vectors(table_name=table_name, records=vectors)
+
+    # Get demo_table row count
+    status, result = milvus.get_table_row_count(table_name)
+
+    # Wait for 6 seconds, since Milvus server persist vector data every 5 seconds by default.
+    # You can set data persist interval in Milvus config file.
+    time.sleep(6)
+
+    # Use the 3rd vector for similarity search
+    query_vectors = [
+        vectors[3]
+    ]
+
+    # execute vector similarity search
+    param = {
+        'table_name': table_name,
+        'query_records': query_vectors,
+        'top_k': 1,
+    }
+    status, results = milvus.search_vectors(**param)
+
+    if results[0][0].score == 100.0 or result[0][0].id == ids[3]:
+        print('Query result is correct')
+    else:
+        print('Query result isn\'t correct')
+
+    # Delete demo_table
+    status = milvus.delete_table(table_name)
+
+    # Disconnect from Milvus
+    status = milvus.disconnect()
+
+
+if __name__ == '__main__':
+    main()

milvus/__init__.py

@@ -2,5 +2,5 @@
 
 __all__ = ['Milvus', 'Prepare', 'Status', 'IndexType', '__version__']
 
-__version__ = '0.1.6'
+__version__ = '0.1.7'
 

milvus/client/Client.py

@@ -1,6 +1,7 @@
 import logging
 import sys
 import struct
+import datetime
 
 from thrift.transport import TSocket
 from thrift.transport import TTransport, TZlibTransport
@@ -23,7 +24,8 @@
 from milvus.client.Exceptions import (
     RepeatingConnectError,
     DisconnectNotConnectedClientError,
-    NotConnectError
+    NotConnectError,
+    ParamError
 )
 from milvus.settings import DefaultConfig as config
 
@@ -34,7 +36,7 @@
 
 LOGGER = logging.getLogger(__name__)
 
-__version__ = '0.1.6'
+__version__ = '0.1.7'
 __NAME__ = 'pymilvus'
 
 
@@ -72,17 +74,29 @@ def table_schema(cls,
                                   store_raw_vector=store_raw_vector)
 
     @classmethod
-    def range(cls, start, end):
+    def range(cls, start_date, end_date):
         """
-        :type start: str
-        :type end: str
-        :param start: (Required) range start
-        :param end: (Required) range end
+        Parser a date or datetime object to Range object
+        :param start_date: start date
+        :type  start_date: datetime or date
+        :param end_date: end date
+        :type  end_date: datetime or date
 
         :return: Range object
         """
-        temp = Range(start=start, end=end)
-        return ttypes.Range(start_value=temp.start, end_value=temp.end)
+        if not (isinstance(start_date, datetime.date) and isinstance(end_date, datetime.date)):
+            raise ParamError('start_date and end_date show be datetime.date object!')
+
+        start = (start_date.year - 1900)*10000 + (start_date.month - 1)*100 + start_date.day
+        end = (end_date.year - 1900)*10000 + (end_date.month - 1)*100 + end_date.day
+        return ttypes.Range(start_value=start, end_value=end)
+
+    @classmethod
+    def ranges(cls, ranges):
+        res = []
+        for _range in ranges:
+            res.append(Prepare.range(_range[0], _range[1]))
+        return res
 
     @classmethod
     def row_record(cls, vector_data):
@@ -239,17 +253,28 @@ def disconnect(self):
     def create_table(self, param):
         """Create table
 
-        :type  param: TableSchema
+        :type  param: dict or TableSchema
         :param param: Provide table information to be created
 
-                `Please use Prepare.table_schema generate param`
+                `example param={'table_name': 'name',
+                                'dimension': 16,
+                                'index_type': IndexType.FLAT,
+                                'store_raw_vector': False}`
+
+                `OR using Prepare.table_schema to create param`
 
         :return: Status, indicate if operation is successful
         :rtype: Status
         """
         if not self.connected:
             raise NotConnectError('Please Connect to the server first!')
 
+        if not isinstance(param, ttypes.TableSchema):
+            if isinstance(param, dict):
+                param = Prepare.table_schema(**param)
+            else:
+                raise ParamError('Param incorrect!')
+
         try:
             self._client.CreateTable(param)
             return Status(message='Table {} created!'.format(param.table_name))
@@ -288,13 +313,15 @@ def add_vectors(self, table_name, records):
         Add vectors to table
 
         :type  table_name: str
-        :type  records: list[RowRecord]
+        :type  records: list[list[float]] or list[RowRecord]
+
+                `example records: [[1.2345],[1.2345]]`
+
+                `OR using Prepare.records`
 
         :param table_name: table name been inserted
         :param records: list of vectors been inserted
 
-                `Please use Prepare.records generate records`
-
         :returns:
             Status: indicate if vectors inserted successfully
 
@@ -304,6 +331,14 @@ def add_vectors(self, table_name, records):
         if not self.connected:
             raise NotConnectError('Please Connect to the server first!')
 
+        if not isinstance(records[0], ttypes.RowRecord):
+            if not records or not records[:1]:
+                raise ParamError('Records empty!')
+            if isinstance(records, list) and isinstance(records[0], list):
+                records = Prepare.records(records)
+            else:
+                raise ParamError('Records param incorrect!')
+
         ids = []
 
         try:
@@ -322,17 +357,23 @@ def search_vectors(self, table_name, top_k, query_records, query_ranges=None):
 
         :param query_ranges: (Optional) ranges for conditional search.
             If not specified, search whole table
-        :type  query_ranges: list[Range]
+        :type  query_ranges: list[tuple(date, date)]
 
-                `Range can be generated by Prepare.range`
+                `example query_ranges:
+                    date_begin1 = datetime.date(2019,1,1),
+                    date_end1 = datetime.date(2019,1,1)
+                    date_begin2 = datetime.datetime.now()
+                    date_end2 = datetime.datetime.now()
+
+                    query_ranges = [(date_begin1, date_end1), (date_begin2, date_end2)]`
 
         :param table_name: table name been queried
         :type  table_name: str
         :param query_records: all vectors going to be queried
 
-                `Please use Prepare.records generate records`
+                `Using Prepare.records generate query_records`
 
-        :type  query_records: list[RowRecord]
+        :type  query_records: list[list[float]] or list[RowRecord]
         :param top_k: int, how many similar vectors will be searched
         :type  top_k: int
 
@@ -347,6 +388,18 @@ def search_vectors(self, table_name, top_k, query_records, query_ranges=None):
         if not self.connected:
             raise NotConnectError('Please Connect to the server first!')
 
+        if not isinstance(query_records[0], ttypes.RowRecord):
+            if not query_records or not query_records[:1]:
+                raise ParamError('query_records empty!')
+            if isinstance(query_records, list) and isinstance(query_records[0], list):
+                query_records = Prepare.records(query_records)
+            else:
+                raise ParamError('query_records param incorrect!')
+
+        if query_ranges:
+            # TODO type check
+            query_ranges = Prepare.ranges(query_ranges)
+
         res = TopKQueryResult()
         try:
             top_k_query_results = self._client.SearchVector(
@@ -372,14 +425,12 @@ def search_vectors_in_files(self, table_name, file_ids, query_records, top_k, qu
         :type  table_name: str
         :param table_name: table name been queried
 
-        :type  file_ids: list[str]
+        :type  file_ids: list[str] or list[int]
         :param file_ids: Specified files id array
 
-        :type  query_records: list[RowRecord]
+        :type  query_records: list[list[float]]
         :param query_records: all vectors going to be queried
 
-                `Please use Prepare.records generate records`
-
         :type  query_ranges: list[Range]
         :param query_ranges: Optional ranges for conditional search.
             If not specified, search whole table
@@ -397,7 +448,17 @@ def search_vectors_in_files(self, table_name, file_ids, query_records, top_k, qu
         if not self.connected:
             raise NotConnectError('Please Connect to the server first!')
 
+        # TODO query_ranges
+        if not isinstance(query_records[0], ttypes.RowRecord):
+            if not query_records or not query_records[:1]:
+                raise ParamError('query_records empty!')
+            if isinstance(query_records, list) and isinstance(query_records[0], list):
+                query_records = Prepare.records(query_records)
+            else:
+                raise ParamError('query_records param incorrect!')
+
         res = TopKQueryResult()
+        file_ids = [str(item) for item in file_ids if isinstance(item, int)]
         try:
             top_k_query_results = self._client.SearchVectorInFiles(
                 table_name=table_name,

milvus/client/Exceptions.py

@@ -14,5 +14,5 @@ class RepeatingConnectError(ConnectError):
     pass
 
 
-class DisconnectNotConnectedClientError(ValueError):
+class DisconnectNotConnectedClientError(ConnectError):
     pass

setup.py

@@ -7,7 +7,7 @@
 
 setuptools.setup(
     name="pymilvus",
-    version="0.1.6",
+    version="0.1.7",
     description="Python Sdk for Milvus",
     long_description=README,
     long_description_content_type='text/markdown',