Adding NomicEmbeddingModel

.github/workflows/release-4-rest.yaml

@@ -45,6 +45,7 @@ jobs:
           HF_API_KEY: ${{ secrets.HF_API_KEY }}
           MILVUS_API_KEY: ${{ secrets.MILVUS_API_KEY }}
           MISTRAL_AI_API_KEY: ${{ secrets.MISTRAL_AI_API_KEY }}
+          NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
           PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
           WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
           WEAVIATE_HOST: ${{ secrets.WEAVIATE_HOST }}

langchain4j-bom/pom.xml

@@ -111,6 +111,12 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>dev.langchain4j</groupId>
+                <artifactId>langchain4j-nomic</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <!-- embedding stores -->
 
             <dependency>

langchain4j-nomic/pom.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>dev.langchain4j</groupId>
+        <artifactId>langchain4j-parent</artifactId>
+        <version>0.27.0-SNAPSHOT</version>
+        <relativePath>../langchain4j-parent/pom.xml</relativePath>
+    </parent>
+
+    <artifactId>langchain4j-nomic</artifactId>
+    <packaging>jar</packaging>
+
+    <name>LangChain4j :: Integration :: Nomic</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.squareup.retrofit2</groupId>
+            <artifactId>retrofit</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.squareup.retrofit2</groupId>
+            <artifactId>converter-gson</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.tinylog</groupId>
+            <artifactId>tinylog-impl</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.tinylog</groupId>
+            <artifactId>slf4j-tinylog</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/EmbeddingRequest.java

@@ -0,0 +1,13 @@
+package dev.langchain4j.model.nomic;
+
+import lombok.Builder;
+
+import java.util.List;
+
+@Builder
+class EmbeddingRequest {
+
+    private String model;
+    private List<String> texts;
+    private String taskType;
+}

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/EmbeddingResponse.java

@@ -0,0 +1,12 @@
+package dev.langchain4j.model.nomic;
+
+import lombok.Getter;
+
+import java.util.List;
+
+@Getter
+class EmbeddingResponse {
+
+    private List<float[]> embeddings;
+    private Usage usage;
+}

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/NomicApi.java

@@ -0,0 +1,14 @@
+package dev.langchain4j.model.nomic;
+
+import retrofit2.Call;
+import retrofit2.http.Body;
+import retrofit2.http.Header;
+import retrofit2.http.Headers;
+import retrofit2.http.POST;
+
+interface NomicApi {
+
+    @POST("embedding/text")
+    @Headers({"Content-Type: application/json"})
+    Call<EmbeddingResponse> embed(@Body EmbeddingRequest request, @Header("Authorization") String authorizationHeader);
+}

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/NomicClient.java

@@ -0,0 +1,73 @@
+package dev.langchain4j.model.nomic;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import lombok.Builder;
+import okhttp3.OkHttpClient;
+import retrofit2.Retrofit;
+import retrofit2.converter.gson.GsonConverterFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+
+import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
+import static dev.langchain4j.internal.ValidationUtils.ensureNotBlank;
+
+class NomicClient {
+
+    private static final Gson GSON = new GsonBuilder()
+            .setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES)
+            .setPrettyPrinting()
+            .create();
+
+    private final NomicApi nomicApi;
+    private final String authorizationHeader;
+
+    @Builder
+    NomicClient(String baseUrl, String apiKey, Duration timeout, Boolean logRequests, Boolean logResponses) {
+
+        OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient.Builder()
+                .callTimeout(timeout)
+                .connectTimeout(timeout)
+                .readTimeout(timeout)
+                .writeTimeout(timeout);
+
+        if (logRequests) {
+            okHttpClientBuilder.addInterceptor(new RequestLoggingInterceptor());
+        }
+        if (logResponses) {
+            okHttpClientBuilder.addInterceptor(new ResponseLoggingInterceptor());
+        }
+
+        Retrofit retrofit = new Retrofit.Builder()
+                .baseUrl(baseUrl)
+                .client(okHttpClientBuilder.build())
+                .addConverterFactory(GsonConverterFactory.create(GSON))
+                .build();
+
+        this.nomicApi = retrofit.create(NomicApi.class);
+        this.authorizationHeader = "Bearer " + ensureNotBlank(apiKey, "apiKey");
+    }
+
+    public EmbeddingResponse embed(EmbeddingRequest request) {
+        try {
+            retrofit2.Response<EmbeddingResponse> retrofitResponse
+                    = nomicApi.embed(request, authorizationHeader).execute();
+
+            if (retrofitResponse.isSuccessful()) {
+                return retrofitResponse.body();
+            } else {
+                throw toException(retrofitResponse);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static RuntimeException toException(retrofit2.Response<?> response) throws IOException {
+        int code = response.code();
+        String body = response.errorBody().string();
+        String errorMessage = String.format("status code: %s; body: %s", code, body);
+        return new RuntimeException(errorMessage);
+    }
+}

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/NomicEmbeddingModel.java

@@ -0,0 +1,77 @@
+package dev.langchain4j.model.nomic;
+
+import dev.langchain4j.data.embedding.Embedding;
+import dev.langchain4j.data.segment.TextSegment;
+import dev.langchain4j.model.embedding.EmbeddingModel;
+import dev.langchain4j.model.output.Response;
+import dev.langchain4j.model.output.TokenUsage;
+import lombok.Builder;
+
+import java.time.Duration;
+import java.util.List;
+
+import static dev.langchain4j.internal.RetryUtils.withRetry;
+import static dev.langchain4j.internal.Utils.getOrDefault;
+import static dev.langchain4j.internal.ValidationUtils.ensureNotBlank;
+import static java.time.Duration.ofSeconds;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * An integration with Nomic Atlas's Text Embeddings API.
+ * See more details <a href="https://docs.nomic.ai/reference/endpoints/nomic-embed-text">here</a>.
+ */
+public class NomicEmbeddingModel implements EmbeddingModel {
+
+    private static final String DEFAULT_BASE_URL = "https://api-atlas.nomic.ai/v1/";
+
+    private final NomicClient client;
+    private final String modelName;
+    private final String taskType;
+    private final Integer maxRetries;
+
+    @Builder
+    public NomicEmbeddingModel(
+            String baseUrl,
+            String apiKey,
+            String modelName,
+            String taskType,
+            Duration timeout,
+            Integer maxRetries,
+            Boolean logRequests,
+            Boolean logResponses
+    ) {
+        this.client = NomicClient.builder()
+                .baseUrl(getOrDefault(baseUrl, DEFAULT_BASE_URL))
+                .apiKey(ensureNotBlank(apiKey, "apiKey"))
+                .timeout(getOrDefault(timeout, ofSeconds(60)))
+                .logRequests(getOrDefault(logRequests, false))
+                .logResponses(getOrDefault(logResponses, false))
+                .build();
+        this.modelName = getOrDefault(modelName, "nomic-embed-text-v1");
+        this.taskType = taskType;
+        this.maxRetries = getOrDefault(maxRetries, 3);
+    }
+
+    public static NomicEmbeddingModel withApiKey(String apiKey) {
+        return NomicEmbeddingModel.builder().apiKey(apiKey).build();
+    }
+
+    @Override
+    public Response<List<Embedding>> embedAll(List<TextSegment> textSegments) {
+
+        EmbeddingRequest request = EmbeddingRequest.builder()
+                .model(modelName)
+                .texts(textSegments.stream().map(TextSegment::text).collect(toList()))
+                .taskType(taskType)
+                .build();
+
+        EmbeddingResponse response = withRetry(() -> client.embed(request), maxRetries);
+
+        List<Embedding> embeddings = response.getEmbeddings().stream()
+                .map(Embedding::from).collect(toList());
+
+        TokenUsage tokenUsage = new TokenUsage(response.getUsage().getTotalTokens(), 0);
+
+        return Response.from(embeddings, tokenUsage);
+    }
+}

langchain4j-nomic/src/main/java/dev/langchain4j/model/nomic/RequestLoggingInterceptor.java

@@ -0,0 +1,82 @@
+package dev.langchain4j.model.nomic;
+
+import okhttp3.Headers;
+import okhttp3.Interceptor;
+import okhttp3.Request;
+import okhttp3.Response;
+import okio.Buffer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static java.util.stream.StreamSupport.stream;
+
+class RequestLoggingInterceptor implements Interceptor {
+
+    private static final Logger log = LoggerFactory.getLogger(RequestLoggingInterceptor.class);
+
+    private static final Pattern BEARER_PATTERN = Pattern.compile("(Bearer\\s*)([\\w-]{5})([\\w-]+)([\\w-]{2})");
+
+    public Response intercept(Interceptor.Chain chain) throws IOException {
+        Request request = chain.request();
+        log(request);
+        return chain.proceed(request);
+    }
+
+    private void log(Request request) {
+        log.debug(
+                "Request:\n" +
+                        "- method: {}\n" +
+                        "- url: {}\n" +
+                        "- headers: {}\n" +
+                        "- body: {}",
+                request.method(),
+                request.url(),
+                inOneLine(request.headers()),
+                getBody(request)
+        );
+    }
+
+    static String inOneLine(Headers headers) {
+        return stream(headers.spliterator(), false)
+                .map((header) -> {
+                    String headerKey = header.component1();
+                    String headerValue = header.component2();
+                    if (headerKey.equals("Authorization")) {
+                        headerValue = maskAuthorizationHeaderValue(headerValue);
+                    }
+                    return String.format("[%s: %s]", headerKey, headerValue);
+                }).collect(Collectors.joining(", "));
+    }
+
+    private static String maskAuthorizationHeaderValue(String authorizationHeaderValue) {
+        try {
+            Matcher matcher = BEARER_PATTERN.matcher(authorizationHeaderValue);
+            StringBuffer sb = new StringBuffer();
+
+            while (matcher.find()) {
+                matcher.appendReplacement(sb, matcher.group(1) + matcher.group(2) + "..." + matcher.group(4));
+            }
+
+            matcher.appendTail(sb);
+            return sb.toString();
+        } catch (Exception e) {
+            return "[failed to mask the API key]";
+        }
+    }
+
+    private static String getBody(Request request) {
+        try {
+            Buffer buffer = new Buffer();
+            request.body().writeTo(buffer);
+            return buffer.readUtf8();
+        } catch (Exception e) {
+            log.warn("Exception happened while reading request body", e);
+            return "[Exception happened while reading request body. Check logs for more details.]";
+        }
+    }
+}