Merge branch 'release/0.7.0' into main

.bazelrc

@@ -3,7 +3,13 @@
 build --strategy=TypeScriptCompile=worker
 
 # Use java 11
-build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 --workspace_status_command=tools/build/bazel_status.sh
+build --java_toolchain=@bazel_tools//tools/jdk:toolchain_java11 --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_java11
+# Remove once annotation processing fix is released (https://github.com/bazelbuild/bazel/issues/12837)
+build --nojava_header_compilation
+
+# Stamp version and commit
+build --stamp --workspace_status_command=tools/build/bazel_status.sh
+
 
 # Output test errors by default
 test --test_output=errors

.bazelversion

@@ -1 +1 @@
-3.5.0
+4.0.0

.github/workflows/main.yml

@@ -54,11 +54,18 @@ jobs:
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
       - name: Upload airy binary to S3
-        if: startsWith(github.ref, 'refs/heads/release') || startsWith(github.ref, 'refs/heads/main') || startsWith(github.ref, 'refs/heads/develop')
+        if: startsWith(github.ref, 'refs/heads/release') || startsWith(github.ref, 'refs/heads/main')
         run: |
           aws s3 cp bazel-bin/infrastructure/cli/airy_linux_bin s3://airy-core-binaries/`cat ./VERSION`/linux/amd64/airy
           aws s3 cp bazel-bin/infrastructure/cli/airy_darwin_bin s3://airy-core-binaries/`cat ./VERSION`/darwin/amd64/airy
           aws s3 cp bazel-bin/infrastructure/cli/airy_windows_bin s3://airy-core-binaries/`cat ./VERSION`/windows/amd64/airy.exe
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Upload airy binary to S3 [develop]
+        if: startsWith(github.ref, 'refs/heads/develop')
+        run: |
+          aws s3 cp bazel-bin/infrastructure/cli/airy_linux_bin s3://airy-core-binaries/develop/linux/amd64/airy
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -1,7 +1,5 @@
 # Editors and IDE's
 
-!.ijwb/.idea/fileTemplates/
-
 .idea/
 .code/
 .vscode/

.prettierignore

@@ -0,0 +1 @@
+docs/docs/cli/*.md

BUILD

@@ -6,6 +6,11 @@ load("@io_bazel_rules_go//go:def.bzl", "TOOLS_NOGO", "nogo")
 
 package(default_visibility = ["//visibility:public"])
 
+alias(
+    name = "tsconfig.json",
+    actual = "//:bazel.tsconfig.json",
+)
+
 multirun(
     name = "fix",
     commands = [
@@ -163,8 +168,8 @@ exports_files(
     [
         "package.json",
         ".prettierrc.json",
+        ".prettierignore",
         "yarn.lock",
-        "tsconfig.json",
     ],
 )
 

README.md

@@ -85,7 +85,7 @@ The bootstrap installation requires
 [Vagrant](https://www.vagrantup.com/downloads) and
 [VirtualBox](https://www.virtualbox.org/wiki/Downloads). If they are not
 found, the script will attempt to install them for you. Check out our [test
-deployment guide](/docs/docs/guides/airy-core-in-test-env.md) for detailed information.
+deployment guide](/docs/docs/getting-started/deployment/test-environment.md) for detailed information.
 
 ## Components
 
@@ -95,9 +95,9 @@ The Airy Core Platform contains the following components:
   Kafka](https://kafka.apache.org) to process incoming webhook data from
   different sources. We make sense of the data and reshape it into source
   independent contacts, conversations, and messages (see our
-  [glossary](/docs/docs/glossary.md) for formal definitions).
+  [glossary](/docs/getting-started/glossary.md) for formal definitions).
 
-- An [API](/docs/docs/api/http.md) to manage the data sets the platform
+- An [API](/docs/docs/api/http/introduction.md) to manage the data sets the platform
   handles.
 
 - A webhook integration server that allows its users to programmatically

VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

WORKSPACE

@@ -10,7 +10,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "com_github_airyhq_bazel_tools",
-    commit = "fd79bd3344b9c95a09eaa94597a49069f943e089",
+    commit = "cdb56bffd21ea94fe909394a2c7321135dfd506f",
     remote = "https://github.com/airyhq/bazel-tools.git",
     shallow_since = "1607079534 +0100",
 )
@@ -34,7 +34,6 @@ maven_install(
         "com.google.auth:google-auth-library-oauth2-http:0.20.0",
         "com.jayway.jsonpath:json-path:2.4.0",
         "com.twilio.sdk:twilio:7.51.0",
-        "cz.habarta.typescript-generator:typescript-generator-core:2.26.723",
         "io.confluent:kafka-avro-serializer:5.5.1",
         "io.confluent:kafka-schema-registry-client:5.5.1",
         "io.confluent:kafka-schema-registry:5.5.1",

backend/api/admin/src/main/java/co/airy/core/api/config/ClientConfigController.java

@@ -4,7 +4,6 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.util.HashMap;
 import java.util.Map;
 
 @RestController

backend/api/admin/src/main/java/co/airy/core/api/config/ServiceDiscovery.java

@@ -1,10 +1,8 @@
 package co.airy.core.api.config;
 
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
-import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;

backend/api/auth/src/main/java/co/airy/core/api/auth/controllers/UsersController.java

@@ -15,10 +15,10 @@
 import co.airy.core.api.auth.dto.User;
 import co.airy.core.api.auth.services.Mail;
 import co.airy.core.api.auth.services.Password;
-import co.airy.spring.web.payload.EmptyResponsePayload;
-import co.airy.spring.web.payload.RequestErrorResponsePayload;
 import co.airy.spring.auth.IgnoreAuthPattern;
 import co.airy.spring.jwt.Jwt;
+import co.airy.spring.web.payload.EmptyResponsePayload;
+import co.airy.spring.web.payload.RequestErrorResponsePayload;
 import org.jdbi.v3.core.statement.UnableToExecuteStatementException;
 import org.springframework.context.annotation.Bean;
 import org.springframework.http.HttpStatus;

backend/api/communication/BUILD

@@ -11,7 +11,6 @@ app_deps = [
     "//backend/model/channel",
     "//backend/model/metadata",
     "//backend:read-receipt",
-    "//lib/java/mapping",
     "//lib/java/date",
     "//lib/java/pagination",
     "//lib/java/spring/auth:spring-auth",

backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java

@@ -1,20 +1,20 @@
 package co.airy.core.api.communication;
 
 import co.airy.avro.communication.Metadata;
-import co.airy.core.api.communication.lucene.ExtendedQueryParser;
-import co.airy.model.metadata.MetadataKeys;
-import co.airy.model.metadata.Subject;
 import co.airy.avro.communication.ReadReceipt;
 import co.airy.core.api.communication.dto.Conversation;
 import co.airy.core.api.communication.dto.ConversationIndex;
 import co.airy.core.api.communication.dto.LuceneQueryResult;
+import co.airy.core.api.communication.lucene.ExtendedQueryParser;
 import co.airy.core.api.communication.lucene.ReadOnlyLuceneStore;
 import co.airy.core.api.communication.payload.ConversationByIdRequestPayload;
 import co.airy.core.api.communication.payload.ConversationListRequestPayload;
 import co.airy.core.api.communication.payload.ConversationListResponsePayload;
 import co.airy.core.api.communication.payload.ConversationResponsePayload;
 import co.airy.core.api.communication.payload.ConversationTagRequestPayload;
 import co.airy.core.api.communication.payload.ResponseMetadata;
+import co.airy.model.metadata.MetadataKeys;
+import co.airy.model.metadata.Subject;
 import co.airy.pagination.Page;
 import co.airy.pagination.Paginator;
 import co.airy.spring.web.payload.EmptyResponsePayload;
@@ -23,7 +23,6 @@
 import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.Query;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -112,7 +111,7 @@ private ResponseEntity<?> queryConversations(ConversationListRequestPayload requ
     private ResponseEntity<ConversationListResponsePayload> listConversations(ConversationListRequestPayload requestPayload) {
         final List<Conversation> conversations = fetchAllConversations();
         int totalSize = conversations.size();
-        conversations.sort(comparing(conversation -> ((Conversation) conversation).getLastMessage().getSentAt()).reversed());
+        conversations.sort(comparing(conversation -> ((Conversation) conversation).getLastMessageContainer().getMessage().getSentAt()).reversed());
 
         final Paginator<Conversation> paginator = new Paginator<>(conversations, Conversation::getId)
                 .from(requestPayload.getCursor()).perPage(requestPayload.getPageSize());

backend/api/communication/src/main/java/co/airy/core/api/communication/Mapper.java

@@ -1,34 +1,26 @@
 package co.airy.core.api.communication;
 
 import co.airy.avro.communication.Message;
-import co.airy.model.metadata.MetadataKeys;
-import co.airy.model.metadata.MetadataRepository;
-import co.airy.model.channel.ChannelPayload;
 import co.airy.core.api.communication.dto.Conversation;
 import co.airy.core.api.communication.dto.DisplayName;
+import co.airy.core.api.communication.dto.MessageContainer;
 import co.airy.core.api.communication.payload.ContactResponsePayload;
 import co.airy.core.api.communication.payload.ConversationResponsePayload;
 import co.airy.core.api.communication.payload.MessageResponsePayload;
-import co.airy.mapping.ContentMapper;
+import co.airy.model.channel.ChannelPayload;
+import co.airy.model.metadata.MetadataKeys;
 import org.springframework.stereotype.Component;
 
 import java.util.Map;
 
-import static co.airy.model.metadata.MetadataRepository.getConversationInfo;
 import static co.airy.date.format.DateFormat.isoFromMillis;
-import static java.util.stream.Collectors.toList;
+import static co.airy.model.metadata.MetadataRepository.getConversationInfo;
+import static co.airy.model.message.MessageRepository.resolveContent;
 
 @Component
 public class Mapper {
-    private final ContentMapper contentMapper;
-
-    Mapper(ContentMapper contentMapper) {
-        this.contentMapper = contentMapper;
-    }
 
     public ConversationResponsePayload fromConversation(Conversation conversation) {
-        final Map<String, String> metadata = conversation.getMetadata();
-
         return ConversationResponsePayload.builder()
                 .channel(ChannelPayload.builder()
                         .id(conversation.getChannelId())
@@ -37,16 +29,10 @@ public ConversationResponsePayload fromConversation(Conversation conversation) {
                         .build())
                 .id(conversation.getId())
                 .unreadMessageCount(conversation.getUnreadMessageCount())
-                .tags(
-                        MetadataRepository.filterPrefix(metadata, MetadataKeys.TAGS)
-                        .keySet()
-                                .stream()
-                                .map(s -> s.split("\\.")[1])
-                                .collect(toList())
-                )
+                .tags(conversation.getTagIds())
                 .createdAt(isoFromMillis(conversation.getCreatedAt()))
                 .contact(getContact(conversation))
-                .lastMessage(fromMessage(conversation.getLastMessage()))
+                .lastMessage(fromMessageContainer(conversation.getLastMessageContainer()))
                 .build();
     }
 
@@ -61,9 +47,10 @@ private ContactResponsePayload getContact(Conversation conversation) {
                 .build();
     }
 
-    public MessageResponsePayload fromMessage(Message message) {
+    public MessageResponsePayload fromMessageContainer(MessageContainer messageContainer) {
+        final Message message = messageContainer.getMessage();
         return MessageResponsePayload.builder()
-                .content(contentMapper.renderWithDefaultAndLog(message))
+                .content(resolveContent(message, messageContainer.getMetadataMap()))
                 .senderType(message.getSenderType().toString().toLowerCase())
                 .deliveryState(message.getDeliveryState().toString().toLowerCase())
                 .id(message.getId())

backend/api/communication/src/main/java/co/airy/core/api/communication/MessagesController.java

@@ -1,6 +1,6 @@
 package co.airy.core.api.communication;
 
-import co.airy.avro.communication.Message;
+import co.airy.core.api.communication.dto.MessageContainer;
 import co.airy.core.api.communication.payload.MessageListRequestPayload;
 import co.airy.core.api.communication.payload.MessageListResponsePayload;
 import co.airy.pagination.Page;
@@ -41,18 +41,19 @@ ResponseEntity<MessageListResponsePayload> messageList(@RequestBody @Valid Messa
     }
 
     private MessageListResponsePayload fetchMessages(String conversationId, int pageSize, String cursor) {
-        final List<Message> messages = stores.getMessages(conversationId);
+        final List<MessageContainer> messages = stores.getMessages(conversationId);
 
         if (messages == null) {
             return null;
         }
 
-        Paginator<Message> paginator = new Paginator<>(messages, Message::getId).perPage(pageSize).from(cursor);
+        Paginator<MessageContainer> paginator = new Paginator<>(messages, (message) -> message.getMessage().getId())
+                .perPage(pageSize).from(cursor);
 
-        Page<Message> page = paginator.page();
+        Page<MessageContainer> page = paginator.page();
 
         return MessageListResponsePayload.builder()
-                .data(page.getData().stream().map(mapper::fromMessage).collect(toList()))
+                .data(page.getData().stream().map(mapper::fromMessageContainer).collect(toList()))
                 .responseMetadata(MessageListResponsePayload.ResponseMetadata.builder()
                         .nextCursor(page.getNextCursor())
                         .previousCursor(cursor)

backend/api/communication/src/main/java/co/airy/core/api/communication/MetadataController.java

@@ -1,9 +1,9 @@
 package co.airy.core.api.communication;
 
 import co.airy.avro.communication.Metadata;
-import co.airy.model.metadata.Subject;
 import co.airy.core.api.communication.payload.RemoveMetadataRequestPayload;
 import co.airy.core.api.communication.payload.SetMetadataRequestPayload;
+import co.airy.model.metadata.Subject;
 import co.airy.spring.web.payload.EmptyResponsePayload;
 import co.airy.spring.web.payload.RequestErrorResponsePayload;
 import org.springframework.http.HttpStatus;

backend/api/communication/src/main/java/co/airy/core/api/communication/SendMessageController.java

@@ -6,6 +6,7 @@
 import co.airy.avro.communication.Message;
 import co.airy.avro.communication.SenderType;
 import co.airy.core.api.communication.dto.Conversation;
+import co.airy.core.api.communication.dto.MessageContainer;
 import co.airy.core.api.communication.payload.SendMessageRequestPayload;
 import co.airy.kafka.schema.application.ApplicationCommunicationMessages;
 import co.airy.spring.web.payload.EmptyResponsePayload;
@@ -72,6 +73,6 @@ public ResponseEntity<?> sendMessage(@RequestBody @Valid SendMessageRequestPaylo
 
         producer.send(new ProducerRecord<>(applicationCommunicationMessages.name(), message.getId(), message)).get();
 
-        return ResponseEntity.ok(mapper.fromMessage(message));
+        return ResponseEntity.ok(mapper.fromMessageContainer(new MessageContainer(message, Map.of())));
     }
 }