Skip to content

Commit

Permalink
Replace FastJson by Gson
Browse files Browse the repository at this point in the history 
Signed-off-by: yhmo <yihua.mo@zilliz.com>
  • Loading branch information
@yhmo
yhmo committed May 22, 2024
1 parent 9f1b9ea commit 7c1a473
Show file tree
Hide file tree
Showing 62 changed files with 1,405 additions and 947 deletions.
120 changes: 62 additions & 58 deletions examples/main/java/io/milvus/BulkWriterExample.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package io.milvus;

import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
import io.milvus.bulkwriter.BulkWriter;
import io.milvus.bulkwriter.CloudImport;
Expand Down Expand Up @@ -74,7 +74,6 @@


public class BulkWriterExample {

// milvus
public static class MilvusConsts {
public static final String HOST = "127.0.0.1";
Expand All @@ -83,6 +82,9 @@ public static class MilvusConsts {
public static final String PASSWORD = "password";
}

private static final Gson GSON_INSTANCE = new Gson();


/**
* If you need to transfer the files generated by bulkWriter to the corresponding remote storage (AWS S3, GCP GCS, Azure Blob, Aliyun OSS, Tencent Cloud TOS),
* you need to configure it accordingly; Otherwise, you can ignore it.
Expand All @@ -95,14 +97,14 @@ public static class StorageConsts {
* please configure the following parameters.
*/
public static final String STORAGE_ENDPOINT = cloudStorage.getEndpoint("http://127.0.0.1:9000");
public static final String STORAGE_BUCKET = "storage.bucket";
public static final String STORAGE_ACCESS_KEY = "storage.access.key";
public static final String STORAGE_SECRET_KEY = "storage.secret.key";
public static final String STORAGE_BUCKET = "a-bucket"; // default bucket name of MinIO/Milvus standalone
public static final String STORAGE_ACCESS_KEY = "minioadmin"; // default ak of MinIO/Milvus standalone
public static final String STORAGE_SECRET_KEY = "minioadmin"; // default sk of MinIO/Milvus standalone
/**
* if using local storage such as Minio
* Please set this parameter to empty.
*/
public static final String STORAGE_REGION = "storage.region";
public static final String STORAGE_REGION = "";

/**
* If using remote storage such as Azure Blob
Expand Down Expand Up @@ -207,15 +209,15 @@ private static void exampleAllTypeCollectionRemote(BulkWriterExample exampleBulk
exampleBulkWriter.retrieveImportData(true);
}

// float vectors + all scalar types, use cloud import api.
// You need to apply a cloud service from Zilliz Cloud(https://zilliz.com/cloud)
for (BulkFileType fileType : fileTypes) {
CollectionSchemaParam collectionSchema = buildAllTypeSchema(false, true);
List<List<String>> batchFiles = exampleBulkWriter.allTypesRemoteWriter(false, collectionSchema, fileType);
exampleBulkWriter.createCollection(ALL_TYPES_COLLECTION_NAME, collectionSchema, false);
exampleBulkWriter.callCloudImport(batchFiles, ALL_TYPES_COLLECTION_NAME);
exampleBulkWriter.retrieveImportData(false);
}
// // float vectors + all scalar types, use cloud import api.
// // You need to apply a cloud service from Zilliz Cloud(https://zilliz.com/cloud)
// for (BulkFileType fileType : fileTypes) {
// CollectionSchemaParam collectionSchema = buildAllTypeSchema(false, true);
// List<List<String>> batchFiles = exampleBulkWriter.allTypesRemoteWriter(false, collectionSchema, fileType);
// exampleBulkWriter.createCollection(ALL_TYPES_COLLECTION_NAME, collectionSchema, false);
// exampleBulkWriter.callCloudImport(batchFiles, ALL_TYPES_COLLECTION_NAME);
// exampleBulkWriter.retrieveImportData(false);
// }
}

private static void localWriter(CollectionSchemaParam collectionSchema, BulkFileType fileType) throws Exception {
Expand All @@ -233,10 +235,10 @@ private static void localWriter(CollectionSchemaParam collectionSchema, BulkFile

// append rows
for (int i = 0; i < 100000; i++) {
JSONObject row = new JSONObject();
row.put("path", "path_" + i);
row.put("vector", GeneratorUtils.genFloatVector(DIM));
row.put("label", "label_" + i);
JsonObject row = new JsonObject();
row.addProperty("path", "path_" + i);
row.add("vector", GSON_INSTANCE.toJsonTree(GeneratorUtils.genFloatVector(DIM)));
row.addProperty("label", "label_" + i);

localBulkWriter.appendRow(row);
}
Expand All @@ -248,7 +250,7 @@ private static void localWriter(CollectionSchemaParam collectionSchema, BulkFile
List<List<String>> batchFiles = localBulkWriter.getBatchFiles();
System.out.printf("Local writer done! output local files: %s%n", batchFiles);
} catch (Exception e) {
System.out.println("localWriter catch exception: " + e);
System.out.println("Local writer catch exception: " + e);
throw e;
}
}
Expand All @@ -262,10 +264,10 @@ private static void remoteWriter(CollectionSchemaParam collectionSchema, BulkFil

// append rows
for (int i = 0; i < 100000; i++) {
JSONObject row = new JSONObject();
row.put("path", "path_" + i);
row.put("vector", GeneratorUtils.genFloatVector(DIM));
row.put("label", "label_" + i);
JsonObject row = new JsonObject();
row.addProperty("path", "path_" + i);
row.add("vector", GSON_INSTANCE.toJsonTree(GeneratorUtils.genFloatVector(DIM)));
row.addProperty("label", "label_" + i);

remoteBulkWriter.appendRow(row);
}
Expand All @@ -278,7 +280,7 @@ private static void remoteWriter(CollectionSchemaParam collectionSchema, BulkFil

System.out.printf("Remote writer done! output remote files: %s%n", batchFiles);
} catch (Exception e) {
System.out.println("remoteWriter catch exception: " + e);
System.out.println("Remote writer catch exception: " + e);
throw e;
}
}
Expand Down Expand Up @@ -348,10 +350,10 @@ public void readRecord(GenericData.Record record) {
private static void appendRow(LocalBulkWriter writer, int begin, int end) {
try {
for (int i = begin; i < end; ++i) {
JSONObject row = new JSONObject();
row.put("path", "path_" + i);
row.put("vector", GeneratorUtils.genFloatVector(DIM));
row.put("label", "label_" + i);
JsonObject row = new JsonObject();
row.addProperty("path", "path_" + i);
row.add("vector", GSON_INSTANCE.toJsonTree(GeneratorUtils.genFloatVector(DIM)));
row.addProperty("label", "label_" + i);

writer.appendRow(row);
if (i % 100 == 0) {
Expand All @@ -371,31 +373,33 @@ private List<List<String>> allTypesRemoteWriter(boolean binVec, CollectionSchema
int batchCount = 10000;

for (int i = 0; i < batchCount; ++i) {
JSONObject rowObject = new JSONObject();
JsonObject rowObject = new JsonObject();

// scalar field
rowObject.put("id", i);
rowObject.put("bool", i % 5 == 0);
rowObject.put("int8", i % 128);
rowObject.put("int16", i % 1000);
rowObject.put("int32", i % 100000);
rowObject.put("float", i / 3);
rowObject.put("double", i / 7);
rowObject.put("varchar", "varchar_" + i);
rowObject.put("json", String.format("{\"dummy\": %s, \"ok\": \"name_%s\"}", i, i));
rowObject.addProperty("id", i);
rowObject.addProperty("bool", i % 5 == 0);
rowObject.addProperty("int8", i % 128);
rowObject.addProperty("int16", i % 1000);
rowObject.addProperty("int32", i % 100000);
rowObject.addProperty("float", i / 3);
rowObject.addProperty("double", i / 7);
rowObject.addProperty("varchar", "varchar_" + i);
rowObject.addProperty("json", String.format("{\"dummy\": %s, \"ok\": \"name_%s\"}", i, i));

// vector field
rowObject.put("vector", binVec ? GeneratorUtils.generatorBinaryVector(128) : GeneratorUtils.generatorFloatValue(128));
rowObject.add("vector",
binVec ? GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorBinaryVector(128).array()) :
GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorFloatValue(128)));

// array field
rowObject.put("arrayInt64", GeneratorUtils.generatorLongValue(10));
rowObject.put("arrayVarchar", GeneratorUtils.generatorVarcharValue(10, 10));
rowObject.put("arrayInt8", GeneratorUtils.generatorInt8Value(10));
rowObject.put("arrayInt16", GeneratorUtils.generatorInt16Value(10));
rowObject.put("arrayInt32", GeneratorUtils.generatorInt32Value(10));
rowObject.put("arrayFloat", GeneratorUtils.generatorFloatValue(10));
rowObject.put("arrayDouble", GeneratorUtils.generatorDoubleValue(10));
rowObject.put("arrayBool", GeneratorUtils.generatorBoolValue(10));
rowObject.add("arrayInt64", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorLongValue(10)));
rowObject.add("arrayVarchar", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorVarcharValue(10, 10)));
rowObject.add("arrayInt8", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorInt8Value(10)));
rowObject.add("arrayInt16", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorInt16Value(10)));
rowObject.add("arrayInt32", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorInt32Value(10)));
rowObject.add("arrayFloat", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorFloatValue(10)));
rowObject.add("arrayDouble", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorDoubleValue(10)));
rowObject.add("arrayBool", GSON_INSTANCE.toJsonTree(GeneratorUtils.generatorBoolValue(10)));

remoteBulkWriter.appendRow(rowObject);
}
Expand Down Expand Up @@ -457,11 +461,11 @@ private static void readCsvSampleData(String filePath, BulkWriter writer) throws
Iterator<CsvDataObject> iterator = csvMapper.readerFor(CsvDataObject.class).with(csvSchema).readValues(csvFile);
while (iterator.hasNext()) {
CsvDataObject dataObject = iterator.next();
JSONObject row = new JSONObject();
JsonObject row = new JsonObject();

row.put("vector", dataObject.toFloatArray());
row.put("label", dataObject.getLabel());
row.put("path", dataObject.getPath());
row.add("vector", GSON_INSTANCE.toJsonTree(dataObject.toFloatArray()));
row.addProperty("label", dataObject.getLabel());
row.addProperty("path", dataObject.getPath());

writer.appendRow(row);
}
Expand All @@ -485,14 +489,14 @@ public String getLabel() {
return label;
}
public List<Float> toFloatArray() {
return new Gson().fromJson(vector, new TypeToken<List<Float>>() {
return GSON_INSTANCE.fromJson(vector, new TypeToken<List<Float>>() {
}.getType());
}
}

private void callBulkInsert(CollectionSchemaParam collectionSchema, List<List<String>> batchFiles) throws InterruptedException {
System.out.println("\n===================== call bulkInsert ====================");
createCollection(ALL_TYPES_COLLECTION_NAME, collectionSchema, false);
createCollection(ALL_TYPES_COLLECTION_NAME, collectionSchema, true);

List<Long> taskIds = new ArrayList<>();
for (List<String> batch : batchFiles) {
Expand All @@ -501,7 +505,7 @@ private void callBulkInsert(CollectionSchemaParam collectionSchema, List<List<St
System.out.println("Create a bulkInert task, task id: " + taskId);
}

while (taskIds.size() > 0) {
while (!taskIds.isEmpty()) {
Iterator<Long> iterator = taskIds.iterator();
List<Long> tempTaskIds = new ArrayList<>();
while (iterator.hasNext()) {
Expand Down Expand Up @@ -688,16 +692,16 @@ private static void exampleCloudImport() throws MalformedURLException {
BulkImportResponse bulkImportResponse = CloudImport.bulkImport(CloudImportConsts.CLOUD_ENDPOINT, CloudImportConsts.API_KEY,
CloudImportConsts.OBJECT_URL, CloudImportConsts.OBJECT_ACCESS_KEY, CloudImportConsts.OBJECT_SECRET_KEY,
CloudImportConsts.CLUSTER_ID, CloudImportConsts.COLLECTION_NAME);
System.out.println(new Gson().toJson(bulkImportResponse));
System.out.println(GSON_INSTANCE.toJson(bulkImportResponse));

System.out.println("\n===================== get import job progress ====================");
String jobId = bulkImportResponse.getJobId();
GetImportProgressResponse getImportProgressResponse = CloudImport.getImportProgress(CloudImportConsts.CLOUD_ENDPOINT, CloudImportConsts.API_KEY, jobId, CloudImportConsts.CLUSTER_ID);
System.out.println(new Gson().toJson(getImportProgressResponse));
System.out.println(GSON_INSTANCE.toJson(getImportProgressResponse));

System.out.println("\n===================== list import jobs ====================");
ListImportJobsResponse listImportJobsResponse = CloudImport.listImportJobs(CloudImportConsts.CLOUD_ENDPOINT, CloudImportConsts.API_KEY, CloudImportConsts.CLUSTER_ID, 10, 1);
System.out.println(new Gson().toJson(listImportJobsResponse));
System.out.println(GSON_INSTANCE.toJson(listImportJobsResponse));
}

private CollectionSchemaParam buildSimpleCollection() {
Expand Down
20 changes: 11 additions & 9 deletions examples/main/java/io/milvus/GeneralExample.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@

package io.milvus;

import com.alibaba.fastjson.JSONObject;
import com.google.protobuf.ByteString;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.milvus.client.MilvusClient;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
Expand Down Expand Up @@ -128,7 +129,7 @@ private boolean hasCollection() {
.build());
CommonUtils.handleResponseStatus(response);
System.out.println(response);
return response.getData().booleanValue();
return response.getData();
}

private R<RpcStatus> loadCollection() {
Expand Down Expand Up @@ -158,7 +159,7 @@ private R<DescribeCollectionResponse> describeCollection() {
.build());
CommonUtils.handleResponseStatus(response);
DescCollResponseWrapper wrapper = new DescCollResponseWrapper(response.getData());
System.out.println(wrapper.toString());
System.out.println(wrapper);
return response;
}

Expand Down Expand Up @@ -416,12 +417,13 @@ private R<MutationResult> insertColumns(String partitionName, int count) {
private R<MutationResult> insertRows(String partitionName, int count) {
System.out.println("========== insertRows() ==========");

List<JSONObject> rowsData = new ArrayList<>();
List<JsonObject> rowsData = new ArrayList<>();
Random ran = new Random();
Gson gson = new Gson();
for (long i = 0L; i < count; ++i) {
JSONObject row = new JSONObject();
row.put(AGE_FIELD, ran.nextInt(99));
row.put(VECTOR_FIELD, CommonUtils.generateFloatVector(VECTOR_DIM));
JsonObject row = new JsonObject();
row.addProperty(AGE_FIELD, ran.nextInt(99));
row.add(VECTOR_FIELD, gson.toJsonTree(CommonUtils.generateFloatVector(VECTOR_DIM)));

rowsData.add(row);
}
Expand Down Expand Up @@ -507,7 +509,7 @@ public static void main(String[] args) {
System.out.println("Search with index");
example.searchFace(searchExpr);

String deleteExpr = ID_FIELD + " in " + deleteIds.toString();
String deleteExpr = ID_FIELD + " in " + deleteIds;
example.delete(partitionName, deleteExpr);
String queryExpr = AGE_FIELD + " == 60";
example.query(queryExpr);
Expand Down
40 changes: 21 additions & 19 deletions examples/main/java/io/milvus/HighLevelExample.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

package io.milvus;

import com.alibaba.fastjson.JSONObject;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.common.collect.Lists;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
Expand Down Expand Up @@ -118,30 +119,31 @@ private R<ListCollectionsResponse> listCollections() {

private R<InsertResponse> insertRows(int rowCount) {
System.out.println("========== high level insertRows ==========");
List<JSONObject> rowsData = new ArrayList<>();
List<JsonObject> rowsData = new ArrayList<>();
Random ran = new Random();
Gson gson = new Gson();
for (long i = 0L; i < rowCount; ++i) {
JSONObject row = new JSONObject();
row.put(AGE_FIELD, ran.nextInt(99));
row.put(VECTOR_FIELD, CommonUtils.generateFloatVector(VECTOR_DIM));
JsonObject row = new JsonObject();
row.addProperty(AGE_FIELD, ran.nextInt(99));
row.add(VECTOR_FIELD, gson.toJsonTree(CommonUtils.generateFloatVector(VECTOR_DIM)));

// $meta if collection EnableDynamicField, you can input this field not exist in schema, else deny
row.put(INT32_FIELD_NAME, ran.nextInt());
row.put(INT64_FIELD_NAME, ran.nextLong());
row.put(VARCHAR_FIELD_NAME, "测试varchar");
row.put(FLOAT_FIELD_NAME, ran.nextFloat());
row.put(DOUBLE_FIELD_NAME, ran.nextDouble());
row.put(BOOL_FIELD_NAME, ran.nextBoolean());
row.addProperty(INT32_FIELD_NAME, ran.nextInt());
row.addProperty(INT64_FIELD_NAME, ran.nextLong());
row.addProperty(VARCHAR_FIELD_NAME, String.format("varchar_%d", i));
row.addProperty(FLOAT_FIELD_NAME, ran.nextFloat());
row.addProperty(DOUBLE_FIELD_NAME, ran.nextDouble());
row.addProperty(BOOL_FIELD_NAME, ran.nextBoolean());

// $json
JSONObject jsonObject = new JSONObject();
jsonObject.put(INT32_FIELD_NAME, ran.nextInt());
jsonObject.put(INT64_FIELD_NAME, ran.nextLong());
jsonObject.put(VARCHAR_FIELD_NAME, "测试varchar");
jsonObject.put(FLOAT_FIELD_NAME, ran.nextFloat());
jsonObject.put(DOUBLE_FIELD_NAME, ran.nextDouble());
jsonObject.put(BOOL_FIELD_NAME, ran.nextBoolean());
row.put(USER_JSON_FIELD, jsonObject);
JsonObject jsonObject = new JsonObject();
jsonObject.addProperty(INT32_FIELD_NAME, ran.nextInt());
jsonObject.addProperty(INT64_FIELD_NAME, ran.nextLong());
jsonObject.addProperty(VARCHAR_FIELD_NAME, String.format("varchar_%d", i));
jsonObject.addProperty(FLOAT_FIELD_NAME, ran.nextFloat());
jsonObject.addProperty(DOUBLE_FIELD_NAME, ran.nextDouble());
jsonObject.addProperty(BOOL_FIELD_NAME, ran.nextBoolean());
row.add(USER_JSON_FIELD, jsonObject);

rowsData.add(row);
}
Expand Down

0 comments on commit 7c1a473

Please sign in to comment.