Merge branch 'master' of https://github.com/Sefaria/Sefaria-Project

Sefaria · May 13, 2022 · bde168a · bde168a
2 parents a38b65e + 5087554
commit bde168a
Show file tree

Hide file tree

Showing 68 changed files with 6,780 additions and 1,811 deletions.
diff --git a/.github/workflows/continuous.yaml b/.github/workflows/continuous.yaml
@@ -11,7 +11,7 @@ jobs:
   duplication-check:
     runs-on: ubuntu-latest
     outputs:
-      inPR: steps.check.outputs.number
+      inPR: ${{ steps.check.outputs.number != '' }}
     steps:
       - name: Check if push is in PR
         id: check
@@ -22,7 +22,7 @@ jobs:
   build-generic:
     runs-on: ubuntu-latest
     needs: duplication-check
-    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && ! needs.duplication-check.outputs.inPR ) }}
+    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }}
     strategy:
       matrix:
         app: [ web, node ]
@@ -72,7 +72,7 @@ jobs:
     needs: 
       - duplication-check
       - build-generic
-    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && ! needs.duplication-check.outputs.inPR ) }}
+    if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }}
     steps:
       - uses: actions/checkout@v2
       - name: Set up QEMU

diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml
@@ -0,0 +1,59 @@
+name: Helm Release
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+
+  changes:
+    permissions:
+      contents: read  # for dorny/paths-filter to fetch a list of changed files
+      pull-requests: read  # for dorny/paths-filter to read pull requests
+    runs-on: ubuntu-latest
+    if: |
+      (github.repository == 'sefaria/sefaria-project')
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+      charts: ${{ steps.filter.outputs.charts }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: |
+            charts:
+              - 'helm-chart/Chart.yaml'
+              - 'helm-chart/**/*'
+
+  chart:
+    name: Release Chart
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # needed to write releases
+    needs:
+      - changes
+    if: |
+      (github.repository == 'sefaria/sefaria-project') && (needs.changes.outputs.charts == 'true')
+    steps:
+      - name: Checkout master
+        uses: actions/checkout@v2
+        with:
+          # Fetch entire history. Required for chart-releaser; see https://github.com/helm/chart-releaser-action/issues/13#issuecomment-602063896
+          fetch-depth: 0
+      - name: Setup
+        shell: bash
+        run: |
+          git config --global user.name "$GITHUB_ACTOR"
+          git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com"
+      - name: Helm Chart Releaser
+        uses: helm/chart-releaser-action@v1.4.0
+        env:
+          CR_SKIP_EXISTING: "false"
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_RELEASE_NAME_TEMPLATE: "helm-chart-{{ .Version }}"
+        with:
+          charts_dir: helm-chart
diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml
@@ -1,7 +1,7 @@
 ---
 apiVersion: v2
 name: sefaria
-version: v0.0.8
+version: v0.0.9
 description: Chart to deploy complete Sefaria environment
 icon: https://raw.githubusercontent.com/Sefaria/Sefaria-Project/e757b59968adbc0d6845eaa1b420f934ad864d32/static/img/logo/icon.svg
 home: https://sefaria.org

diff --git a/helm-chart/templates/configmap/create-mongo-dumps.yaml b/helm-chart/templates/configmap/create-mongo-dumps.yaml
@@ -10,33 +10,40 @@ data:
   create-dumps.sh: |-
     #!/usr/bin/env bash
 
+    {{- if eq .Values.sandbox "true" }}
+    DATABASE="${SEFARIA_DB}-{{ .Values.deployEnv | quote }}"
+    {{- else }}
+    DATABASE="${SEFARIA_DB}"
+    {{- end }}
+
+
     DATADIR="/mongodumps/shared_volume"
     echo "dumping"
-    until mongodump --host=mongo:27017 -vvv -d sefaria --collection history -o "${DATADIR}/dump"
+    until mongodump --host=mongo:27017 -vvv -d $DATABASE --collection history -o "${DATADIR}/dump"
     do
     echo "trying to dump history again"
     sleep 2
     done
 
-    until mongodump --host=mongo:27017 -v -d sefaria --collection texts -o "${DATADIR}/dump"
+    until mongodump --host=mongo:27017 -v -d $DATABASE --collection texts -o "${DATADIR}/dump"
     do
     echo "trying to dump texts again"
     sleep 2
     done
 
-    until mongodump --host=mongo:27017 -v -d sefaria --collection links -o "${DATADIR}/dump"
+    until mongodump --host=mongo:27017 -v -d $DATABASE --collection links -o "${DATADIR}/dump"
     do
     echo "trying to dump links again"
     sleep 2
     done
 
-    until mongodump --host=mongo:27017 -v -d sefaria --collection sheets -o "${DATADIR}/dump"
+    until mongodump --host=mongo:27017 -v -d $DATABASE --collection sheets -o "${DATADIR}/dump"
     do
     echo "trying to dump sheets again"
     sleep 2
     done
 
-    until mongodump --host=mongo:27017 -v -d sefaria --excludeCollection=history --excludeCollection=texts --excludeCollection=sheets --excludeCollection=links --excludeCollection=user_history -o "${DATADIR}/dump"
+    until mongodump --host=mongo:27017 -v -d $DATABASE --excludeCollection=history --excludeCollection=texts --excludeCollection=sheets --excludeCollection=links --excludeCollection=user_history -o "${DATADIR}/dump"
     do
     echo "trying to dump other stuff again"
     sleep 2

diff --git a/helm-chart/templates/configmap/local-settings.yaml b/helm-chart/templates/configmap/local-settings.yaml
@@ -225,6 +225,16 @@ data:
 
     MOBILE_APP_KEY = os.getenv("MOBILE_APP_KEY")
 
+    RAW_REF_MODEL_BY_LANG_FILEPATH = {
+        "en": None,
+        "he": None
+    }
+
+    RAW_REF_PART_MODEL_BY_LANG_FILEPATH = {
+        "en": None,
+        "he": None
+    }
+
     LOGGING = {
         'version': 1,
         'disable_existing_loggers': False,

diff --git a/helm-chart/templates/configmap/mongo-restore.yaml b/helm-chart/templates/configmap/mongo-restore.yaml
@@ -17,5 +17,5 @@ data:
     set -x
 
     tar xzvf /storage/dump.tar.gz -C /storage 
-    mongorestore --drop --host=$MONGO_HOST:27017 -v -d sefaria-{{ .Values.deployEnv }} --dir=/storage/dump/sefaria
+    mongorestore --drop --host=$MONGO_HOST:27017 -v -d "${SEFARIA_DB}-{{ .Values.deployEnv }}" --dir=/storage/dump/sefaria
 {{- end }}
diff --git a/helm-chart/templates/cronjob/mongo-backup.yaml b/helm-chart/templates/cronjob/mongo-backup.yaml
@@ -19,9 +19,11 @@ spec:
           - name: create-dumps-script
             configMap:
               name: create-dumps-{{ .Values.deployEnv }}
+              defaultMode: 0755
           - name: upload-dumps-script
             configMap:
               name: upload-dumps-{{ .Values.deployEnv }}
+              defaultMode: 0755
           - name: shared-volume
             emptyDir: {}
           affinity:
@@ -42,6 +44,15 @@ spec:
           initContainers:
           - name: mongo-dumper
             image:  mongo:4.4
+            envFrom:
+              - secretRef:
+                  name: {{ .Values.secrets.localSettings.ref }}
+                  optional: true
+              - configMapRef:
+                  name: local-settings-{{ .Values.deployEnv }}
+              - secretRef:
+                  name: local-settings-secrets-{{ .Values.deployEnv }}
+                  optional: true
             volumeMounts:
             - name: shared-volume
               mountPath: /mongodumps/shared_volume

diff --git a/helm-chart/templates/jobs/mongo-restore.yaml b/helm-chart/templates/jobs/mongo-restore.yaml
@@ -52,6 +52,13 @@ spec:
           env:
             - name: MONGO_HOST
               value: {{ .Values.localSettings.MONGO_HOST | quote }}
+          envFrom:
+             - secretRef:
+                 name: {{ .Values.secrets.localSettings.ref }}
+                 optional: true
+             - secretRef:
+                 name: local-settings-secrets-{{ .Values.deployEnv }}
+                 optional: true
           command: ["/script/restore-mongo.sh"]
       restartPolicy: Never
   ttlSecondsAfterFinished: 432000

diff --git a/node/webpack.config.js b/node/webpack.config.js
@@ -225,4 +225,15 @@ var categorizeSheetsConfig = config({
 
 })
 
-module.exports = [clientConfig, serverConfig, diffConfig, exploreConfig, sefariajsConfig, jsonEditorConfig, timelineConfig, categorizeSheetsConfig];
+var linkerV3Config = config({
+    context: path.resolve('./static/js'),
+    entry: './linker.v3/main',
+    mode: 'development', // can be overriden via cli
+    output: {
+        path: path.resolve(buildDir + 'linker.v3'),
+        filename: 'linker.v3.js'
+    }
+
+})
+
+module.exports = [clientConfig, serverConfig, diffConfig, exploreConfig, sefariajsConfig, jsonEditorConfig, timelineConfig, categorizeSheetsConfig, linkerV3Config];