Merge branch 'main' into improvement/new-sentiment

CrowdDotDev · Nov 13, 2023 · a1f7fb9 · a1f7fb9
2 parents 3ab9cb0 + 05bebcb
commit a1f7fb9
Show file tree

Hide file tree

Showing 142 changed files with 62,404 additions and 23,351 deletions.
diff --git a/.github/workflows/lf-production-deploy-new.yaml b/.github/workflows/lf-production-deploy-new.yaml
@@ -7,6 +7,10 @@ on:
         description: Deploy search-sync-worker service?
         required: true
         type: boolean
+      deploy_search_sync_api:
+          description: Deploy search-sync-api service?
+          required: true
+          type: boolean
       deploy_integration_sync_worker:
         description: Deploy integration-sync-worker service?
         required: true
@@ -56,6 +60,27 @@ jobs:
       - name: Set docker image output
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  build-and-push-search-sync-api:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_search_sync_api }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: search-sync-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
   build-and-push-integration-sync-worker:
     runs-on: ubuntu-latest
@@ -159,6 +184,24 @@ jobs:
           image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
 
+  deploy-search-sync-api:
+    needs: build-and-push-search-sync-api
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_search_sync_api }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: search-sync-api
+          image: ${{ needs.build-and-push-search-sync-api.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
   deploy-integration-sync-worker:
     needs: build-and-push-integration-sync-worker
     runs-on: ubuntu-latest

diff --git a/.github/workflows/lf-staging-deploy-search-sync-api.yaml b/.github/workflows/lf-staging-deploy-search-sync-api.yaml
@@ -0,0 +1,60 @@
+name: LF Staging Deploy Search Sync API
+
+on:
+  push:
+    branches:
+      - 'lf-staging/**'
+      - 'lf-staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/search_sync_api/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.LF_STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.LF_STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.LF_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.LF_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.LF_AWS_REGION }}
+  SLACK_CHANNEL: deploys-lf-staging
+  SLACK_WEBHOOK: ${{ secrets.LF_STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: search-sync-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-search-sync-api:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: search-sync-api
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
diff --git a/.github/workflows/production-deploy-new.yaml b/.github/workflows/production-deploy-new.yaml
@@ -7,6 +7,10 @@ on:
         description: Deploy search-sync-worker service?
         required: true
         type: boolean
+      deploy_search_sync_api:
+        description: Deploy search-sync-api service?
+        required: true
+        type: boolean
       deploy_integration_sync_worker:
         description: Deploy integration-sync-worker service?
         required: true
@@ -56,6 +60,27 @@ jobs:
       - name: Set docker image output
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  build-and-push-search-sync-api:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_search_sync_api }}
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: search-sync-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
 
   build-and-push-integration-sync-worker:
     runs-on: ubuntu-latest
@@ -159,6 +184,24 @@ jobs:
           image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
 
+  deploy-search-sync-api:
+    needs: build-and-push-search-sync-api
+    runs-on: ubuntu-latest
+    if: ${{ inputs.deploy_search_sync_api }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: search-sync-api
+          image: ${{ needs.build-and-push-search-sync-api.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
   deploy-integration-sync-worker:
     needs: build-and-push-integration-sync-worker
     runs-on: ubuntu-latest

diff --git a/.github/workflows/staging-deploy-search-sync-api.yaml b/.github/workflows/staging-deploy-search-sync-api.yaml
@@ -0,0 +1,60 @@
+name: Staging Deploy Search Sync API
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+      - 'staging-**'
+    paths:
+      - 'services/libs/**'
+      - 'services/apps/search_sync_api/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  CROWD_ROLE_ARN: ${{ secrets.STAGING_CLUSTER_ROLE_ARN }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  SLACK_CHANNEL: deploys-staging
+  SLACK_WEBHOOK: ${{ secrets.STAGING_SLACK_CHANNEL_HOOK }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: search-sync-api
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-search-sync-api:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: search-sync-api
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
diff --git a/README.md b/README.md
@@ -30,14 +30,14 @@
 
 ## Table of Contents
 - [About crowd.dev](#about-crowddev)
-- [Features](#✨-features)
-- [Getting started](#🚀-getting-started)
-- [Roadmap](#🗺️-roadmap)
-- [Stay up-to-date](#🔔-stay-up-to-date)
-- [Contribution](#✍️-contribution)
-- [License](#⚖️-license)
-- [Security](#🔒-security)
-- [Book a call](#📞-book-a-call)
+- [Features](#features)
+- [Getting started](#getting-started)
+- [Roadmap](#roadmap)
+- [Stay up-to-date](#stay-up-to-date)
+- [Contribution](#contribution)
+- [License](#license)
+- [Security](#security)
+- [Book a call](#book-a-call)
 
 ## About crowd.dev
 crowd.dev is the Developer Data Platform(DDP) that allows companies to centralize all touch points developers have with their product and brand, whether in the community (e.g., Stack Overflow or Reddit), product (open-source or SaaS), or commercial channels (e.g., HubSpot).The platform pulls data from various sources, normalizes it, matches identities across platforms, and enriches it with third-party data. The result is a unified 360-degree view of the developers who engage with your product and community, the companies they work for, and their position in their personal customer journey. 
@@ -54,7 +54,7 @@ crowd.dev is open-source, built with developers in mind, available for both host
 - Our product is built for extensibility. If you can think of any use cases that you want to build with the data we collect and store for you, please go ahead and build them! We will be here to help out if you need us.
 - You can actively contribute to crowd.dev (e.g. integrations), and we will be supporting you along the journey. Just take a look at our [Contributing guide](https://github.com/CrowdDotDev/crowd.dev/blob/main/CONTRIBUTING.md).
 
-## ✨ Features
+## Features
 
 - **Plug & play integrations** to tie all relevant platforms - like GitHub, Discord, Slack, or LinkedIn - together. ([all integrations](https://www.crowd.dev/integrations))
 - **Identity resolution & automated segmentation** to effortlessly understand activities and profiles across platforms.
@@ -67,7 +67,7 @@ crowd.dev is open-source, built with developers in mind, available for both host
 - **[Eagle Eye](https://www.crowd.dev/eagle-eye)**: Monitor dev-focused community platforms to find relevant content to engage with, helping you to gain developers’ mindshare and grow your community organically [cloud only]
 
 
-## 🚀 Getting started
+## Getting started
 
 ### Cloud version
 
@@ -118,17 +118,17 @@ This app will be available at http://localhost:8081
 
 For more information on development, you can <a href="https://docs.crowd.dev/docs/docker-compose-single-machine-development-with-docker-images">check our docs</a>.
 
-## 🗺️ Roadmap
+## Roadmap
 
 You can find more features on our [public roadmap](https://crowd.dev/roadmap). Feel free to also [open an issue](https://crowd.dev/open-an-issue) for anything you're missing.
 
 
-## 🔔 Stay up-to-date
+## Stay up-to-date
 
 crowd.dev is still in beta and we ship new features every week. To stay in the loop, leave us a star and subscribe to our <a href="https://crowd.dev/newsletter-sign-up">monthly newsletter</a>. Thanks a lot! ❤️
 
 
-## ✍️ Contribution
+## Contribution
 
 There are many ways you can contribute to crowd.dev! Here are a few options:
 
@@ -140,17 +140,17 @@ If you would like to contribute to the development of the project, please refer
 
 All contributions are highly appreciated. 🙏
 
-## ⚖️ License
+## License
 
 Distributed under the Apache 2.0 License. See `LICENSE` for more information.
 
 Our self-hosted version can be run and deployed by default under the permissive Apache 2.0 license. All premium components will be hidden and inactive with the default configuration. You can run, deploy, and contribute to the app without fearing a violation of the premium license. Check out the [premium self-hosted features docs](https://docs.crowd.dev/docs/premium-self-hosted-apps) to know more about the premium self-hosted features.
 
-## 🔒 Security
+## Security
 
 We take security very seriously. If you come across any security vulnerabilities, please disclose them by sending an email to security@crowd.dev. We appreciate your help in making our platform as secure as possible and are committed to working with you to resolve any issues quickly and efficiently.
 
-## 📞 Book a call
+## Book a call
 
 Schedule a call with a crowd.dev team member to learn more about our product and ensure you get the most out of it.
 

diff --git a/backend/.env.dist.composed b/backend/.env.dist.composed
@@ -28,4 +28,7 @@ CROWD_OPENSEARCH_NODE=http://open-search:9200
 CROWD_TEMPORAL_SERVER_URL=temporal:7233
 
 # HuggingFace
-CROWD_HUGGINGFACE_API_KEY=
+CROWD_HUGGINGFACE_API_KEY=
+
+# Seach sync api
+CROWD_SEARCH_SYNC_API_URL=http://search-sync-api:8083
diff --git a/backend/.env.dist.local b/backend/.env.dist.local
@@ -172,4 +172,7 @@ CROWD_TEMPORAL_SERVER_URL=localhost:7233
 CROWD_TEMPORAL_NAMESPACE=default
 
 # HuggingFace
-CROWD_HUGGINGFACE_API_KEY=
+CROWD_HUGGINGFACE_API_KEY=
+
+# Seach sync api
+CROWD_SEARCH_SYNC_API_URL=http://search-sync-api:8083
diff --git a/backend/.env.test b/backend/.env.test
@@ -8,4 +8,4 @@ CROWD_REDIS_PORT=6380
 CROWD_SQS_PORT=9325
 
 # Temporal settings
-CROWD_TEMPORAL_SERVER_URL=localhost:7234
+CROWD_TEMPORAL_SERVER_URL=localhost:7234
diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json
@@ -201,5 +201,8 @@
     "namespace": "CROWD_TEMPORAL_NAMESPACE",
     "certificate": "CROWD_TEMPORAL_CERTIFICATE",
     "privateKey": "CROWD_TEMPORAL_PRIVATE_KEY"
+  },
+  "searchSyncApi": {
+    "baseUrl": "CROWD_SEARCH_SYNC_API_URL"
   }
 }
diff --git a/backend/config/default.json b/backend/config/default.json
@@ -59,5 +59,6 @@
   },
   "temporal": {
     "automationsTaskQueue": "automations"
-  }
+  },
+  "searchSyncApi": {}
 }