From 30af489c14c4b3544d873c013068847dd3e1136d Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Mon, 4 Apr 2022 12:51:15 +0200
Subject: [PATCH 01/13] redis ssl cleanup

---
 docker-compose-dataworker.yml                       | 1 -
 docker-compose.yml                                  | 7 -------
 server/workers/api/src/apis/utils.py                | 3 +--
 server/workers/base/run_base.py                     | 3 +--
 server/workers/dataprocessing/run_dataprocessing.py | 3 +--
 server/workers/flavorconfigs/example.env            | 1 -
 server/workers/gsheets/run_gsheets.py               | 3 +--
 server/workers/openaire/run_openaire.py             | 3 +--
 server/workers/pubmed/run_pubmed.py                 | 3 +--
 9 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/docker-compose-dataworker.yml b/docker-compose-dataworker.yml
index 313ca5295..9ca1e3804 100644
--- a/docker-compose-dataworker.yml
+++ b/docker-compose-dataworker.yml
@@ -12,7 +12,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
diff --git a/docker-compose.yml b/docker-compose.yml
index 69425a221..caaa1b230 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -52,7 +52,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       REDIS_DB: "${REDIS_DB}"
-      REDIS_SSL: "${REDIS_SSL}"
       BEHIND_PROXY: "${BEHIND_PROXY}"
       DEFAULT_DATABASE: "${DEFAULT_DATABASE}"
       DATABASES: "${DATABASES}"
@@ -90,7 +89,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       TRIPLE_USER: "${TRIPLE_USER}"
       TRIPLE_PASS: "${TRIPLE_PASS}"
@@ -115,7 +113,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
     restart: always
     depends_on:
@@ -133,7 +130,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
     restart: always
     volumes:
@@ -152,7 +148,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       LOGFILE: "/var/log/headstart/headstart.log"
       RENV_VERSION: 0.14.0-5
@@ -178,7 +173,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       LOGFILE: "/var/log/headstart/headstart.log"
       RENV_VERSION: 0.14.0-5
@@ -204,7 +198,6 @@ services:
       REDIS_PORT: "${REDIS_PORT}"
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      REDIS_SSL: "${REDIS_SSL}"
       LOGLEVEL: "${LOGLEVEL}"
       LOGFILE: "/var/log/headstart/headstart.log"
       RENV_VERSION: 0.14.0-5
diff --git a/server/workers/api/src/apis/utils.py b/server/workers/api/src/apis/utils.py
index 05b7bcd8e..bfc4167b6 100644
--- a/server/workers/api/src/apis/utils.py
+++ b/server/workers/api/src/apis/utils.py
@@ -11,8 +11,7 @@
     "port": os.getenv("REDIS_PORT"),
     "db": os.getenv("REDIS_DB"),
     "password": os.getenv("REDIS_PASSWORD"),
-    "client_name": "api",
-    "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+    "client_name": "api"
 }
 redis_store = redis.StrictRedis(**redis_config)
 
diff --git a/server/workers/base/run_base.py b/server/workers/base/run_base.py
index f0c4b08f3..93f8146bc 100644
--- a/server/workers/base/run_base.py
+++ b/server/workers/base/run_base.py
@@ -10,8 +10,7 @@
         "port": os.getenv("REDIS_PORT"),
         "db": os.getenv("REDIS_DB"),
         "password": os.getenv("REDIS_PASSWORD"),
-        "client_name": "base_retrieval",
-        "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+        "client_name": "base_retrieval"
     }
 
     redis_store = redis.StrictRedis(**redis_config)
diff --git a/server/workers/dataprocessing/run_dataprocessing.py b/server/workers/dataprocessing/run_dataprocessing.py
index 375f8b603..e64a488d8 100644
--- a/server/workers/dataprocessing/run_dataprocessing.py
+++ b/server/workers/dataprocessing/run_dataprocessing.py
@@ -10,8 +10,7 @@
         "port": os.getenv("REDIS_PORT"),
         "db": os.getenv("REDIS_DB"),
         "password": os.getenv("REDIS_PASSWORD"),
-        "client_name": "dataprocessing",
-        "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+        "client_name": "dataprocessing"
     }
 
     redis_store = redis.StrictRedis(**redis_config)
diff --git a/server/workers/flavorconfigs/example.env b/server/workers/flavorconfigs/example.env
index ded6b8eec..d9c1e8133 100644
--- a/server/workers/flavorconfigs/example.env
+++ b/server/workers/flavorconfigs/example.env
@@ -13,7 +13,6 @@ REDIS_HOST=stable_redis_1
 REDIS_PORT=6379
 REDIS_DB=0
 REDIS_PASSWORD=redis_password
-REDIS_SSL=false
 LOGLEVEL=INFO
 
 BEHIND_PROXY=True
diff --git a/server/workers/gsheets/run_gsheets.py b/server/workers/gsheets/run_gsheets.py
index 4f507a4b7..63b1aef09 100644
--- a/server/workers/gsheets/run_gsheets.py
+++ b/server/workers/gsheets/run_gsheets.py
@@ -10,8 +10,7 @@
         "port": os.getenv("REDIS_PORT"),
         "db": os.getenv("REDIS_DB"),
         "password": os.getenv("REDIS_PASSWORD"),
-        "client_name": "gsheets_retrieval",
-        "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+        "client_name": "gsheets_retrieval"
     }
     redis_store = redis.StrictRedis(**redis_config)
     gc = GSheetsClient(redis_store, os.environ.get("LOGLEVEL", "INFO"))
diff --git a/server/workers/openaire/run_openaire.py b/server/workers/openaire/run_openaire.py
index e752bf1b3..9bb54216e 100644
--- a/server/workers/openaire/run_openaire.py
+++ b/server/workers/openaire/run_openaire.py
@@ -10,8 +10,7 @@
         "port": os.getenv("REDIS_PORT"),
         "db": os.getenv("REDIS_DB"),
         "password": os.getenv("REDIS_PASSWORD"),
-        "client_name": "openaire_retrieval",
-        "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+        "client_name": "openaire_retrieval"
     }
     redis_store = redis.StrictRedis(**redis_config)
     wrapper = OpenAIREClient("./other-scripts", "run_openaire.R", redis_store,
diff --git a/server/workers/pubmed/run_pubmed.py b/server/workers/pubmed/run_pubmed.py
index c4ab40ba5..4d1e734ef 100644
--- a/server/workers/pubmed/run_pubmed.py
+++ b/server/workers/pubmed/run_pubmed.py
@@ -10,8 +10,7 @@
         "port": os.getenv("REDIS_PORT"),
         "db": os.getenv("REDIS_DB"),
         "password": os.getenv("REDIS_PASSWORD"),
-        "client_name": "pubmed_retrieval",
-        "ssl": True if os.getenv("REDIS_SSL") == "true" else False
+        "client_name": "pubmed_retrieval"
     }
     redis_store = redis.StrictRedis(**redis_config)
     wrapper = PubMedClient("./other-scripts", "run_pubmed.R", redis_store,

From 1a862a895fa737b3c40e8c1596ad58fd0eed1a6c Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Mon, 4 Apr 2022 12:51:50 +0200
Subject: [PATCH 02/13] rwrapper function renaming

---
 server/workers/base/src/base.py                | 10 ++++++++--
 server/workers/common/r_wrapper.py             |  4 ++--
 server/workers/dataprocessing/src/headstart.py |  6 +++---
 server/workers/openaire/src/openaire.py        |  4 ++--
 server/workers/pubmed/src/pubmed.py            |  4 ++--
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py
index 904b0fdad..05577d32c 100644
--- a/server/workers/base/src/base.py
+++ b/server/workers/base/src/base.py
@@ -20,7 +20,7 @@ def next_item(self):
         endpoint = msg.get('endpoint')
         return k, params, endpoint
 
-    def execute_r(self, params):
+    def execute_search(self, params):
         q = params.get('q')
         service = params.get('service')
         data = {}
@@ -52,6 +52,8 @@ def execute_r(self, params):
             self.logger.error(error)
             raise
 
+    def execute_contentproviders(self):
+
     def run(self):
         while True:
             k, params, endpoint = self.next_item()
@@ -59,7 +61,7 @@ def run(self):
             self.logger.debug(params)
             if endpoint == "search":
                 try:
-                    res = self.execute_r(params)
+                    res = self.execute_search(params)
                     res["id"] = k
                     if res.get("status") == "error" or params.get('raw') is True:
                         self.redis_store.set(k+"_output", json.dumps(res))
@@ -68,3 +70,7 @@ def run(self):
                 except Exception as e:
                     self.logger.exception("Exception during data retrieval.")
                     self.logger.error(params)
+
+            if endpoint == "contentproviders":
+                try:
+                    res = self.execute
diff --git a/server/workers/common/r_wrapper.py b/server/workers/common/r_wrapper.py
index 5aaf8b887..cc3825e71 100644
--- a/server/workers/common/r_wrapper.py
+++ b/server/workers/common/r_wrapper.py
@@ -37,7 +37,7 @@ def add_default_params(self, params):
     def next_item(self):
         raise NotImplementedError
 
-    def execute_r(self, params):
+    def execute_search(self, params):
         raise NotImplementedError
 
     def run(self):
@@ -49,7 +49,7 @@ def run(self):
                 try:
                     res = {}
                     res["id"] = k
-                    res["input_data"] = self.execute_r(params)
+                    res["input_data"] = self.execute_search(params)
                     res["params"] = params
                     if params.get('raw') is True:
                         redis_store.set(k+"_output", json.dumps(res))
diff --git a/server/workers/dataprocessing/src/headstart.py b/server/workers/dataprocessing/src/headstart.py
index f2b15bda2..4923be7c3 100644
--- a/server/workers/dataprocessing/src/headstart.py
+++ b/server/workers/dataprocessing/src/headstart.py
@@ -52,7 +52,7 @@ def next_item(self):
         input_data = msg.get('input_data')
         return k, params, input_data
 
-    def execute_r(self, params, input_data):
+    def execute_search(self, params, input_data):
         q = params.get('q')
         service = params.get('service')
         data = {}
@@ -92,7 +92,7 @@ def run(self):
                 if params.get('vis_type') == "timeline":
                     # the step of create_map can be dropped once deduplication is possible in API backend as well
                     # TODO: create deduplicate endpoint in service worker and connect to that
-                    metadata = self.execute_r(params, input_data)
+                    metadata = self.execute_search(params, input_data)
                     sg_data = sg.get_streamgraph_data(json.loads(metadata),
                                                     params.get('q'),
                                                     params.get('top_n', 12),
@@ -103,7 +103,7 @@ def run(self):
                     res["status"] = "success"
                     self.redis_store.set(k+"_output", json.dumps(res))
                 else:
-                    res = self.execute_r(params, input_data)
+                    res = self.execute_search(params, input_data)
                     self.redis_store.set(k+"_output", json.dumps(res))
             except ValueError as e:
                 self.logger.error(params)
diff --git a/server/workers/openaire/src/openaire.py b/server/workers/openaire/src/openaire.py
index 7538d4d00..cc774eb7e 100644
--- a/server/workers/openaire/src/openaire.py
+++ b/server/workers/openaire/src/openaire.py
@@ -19,7 +19,7 @@ def next_item(self):
         endpoint = msg.get('endpoint')
         return k, params, endpoint
 
-    def execute_r(self, params):
+    def execute_search(self, params):
         q = params.get('acronym')
         service = params.get('service')
         data = {}
@@ -52,7 +52,7 @@ def run(self):
                 try:
                     res = {}
                     res["id"] = k
-                    res["input_data"] = self.execute_r(params)
+                    res["input_data"] = self.execute_search(params)
                     res["params"] = params
                     if params.get('raw') is True:
                         self.redis_store.set(k+"_output", json.dumps(res))
diff --git a/server/workers/pubmed/src/pubmed.py b/server/workers/pubmed/src/pubmed.py
index c8c792003..865f82ab8 100644
--- a/server/workers/pubmed/src/pubmed.py
+++ b/server/workers/pubmed/src/pubmed.py
@@ -19,7 +19,7 @@ def next_item(self):
         endpoint = msg.get('endpoint')
         return k, params, endpoint
 
-    def execute_r(self, params):
+    def execute_search(self, params):
         q = params.get('q')
         service = params.get('service')
         data = {}
@@ -52,7 +52,7 @@ def run(self):
                 try:
                     res = {}
                     res["id"] = k
-                    res["input_data"] = self.execute_r(params)
+                    res["input_data"] = self.execute_search(params)
                     res["params"] = params
                     if params.get('raw') is True:
                         self.redis_store.set(k+"_output", json.dumps(res))

From b813d28662ea62e18eb8e66206164cdd4a1dfe17 Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Mon, 4 Apr 2022 13:02:46 +0200
Subject: [PATCH 03/13] first endpoint draft

---
 .../other-scripts/run_base_contentproviders.R | 44 +++++++++++++++++++
 server/workers/base/src/base.py               |  3 +-
 2 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 server/preprocessing/other-scripts/run_base_contentproviders.R

diff --git a/server/preprocessing/other-scripts/run_base_contentproviders.R b/server/preprocessing/other-scripts/run_base_contentproviders.R
new file mode 100644
index 000000000..d0eaa6507
--- /dev/null
+++ b/server/preprocessing/other-scripts/run_base_contentproviders.R
@@ -0,0 +1,44 @@
+rm(list = ls())
+
+args <- commandArgs(TRUE)
+wd <- args[1]
+setwd(wd) #Don't forget to set your working directory
+
+renv::activate()
+renv::restore(lockfile = '../renv.lock')
+Sys.setlocale(category="LC_ALL", locale = "en_US.UTF-8")
+
+library(jsonlite)
+library(logging)
+
+source('utils.R')
+if (Sys.getenv("LOGLEVEL") == "DEBUG") {
+  DEBUG <- FALSE
+} else {
+  DEBUG <- TRUE
+}
+
+if (DEBUG==TRUE){
+  setup_logging('DEBUG')
+} else {
+  setup_logging('INFO')
+}
+
+
+log <- getLogger('base_repos')
+
+tryCatch({
+  contentproviders <- bs_repositories("")
+}, error=function(err){
+  log$error(paste("Contentprovider failed", "base", "retrieve_contentproviders", "", err, sep="||"))
+  failed <- list()
+  failed$reason <- list(err)
+  failed$status <- 'error'
+})
+
+
+if (exists('contentproviders')) {
+  print(toJSON(contentproviders))
+} else {
+  print(toJSON(failed))
+}
diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py
index 05577d32c..a02d87f1b 100644
--- a/server/workers/base/src/base.py
+++ b/server/workers/base/src/base.py
@@ -52,7 +52,8 @@ def execute_search(self, params):
             self.logger.error(error)
             raise
 
-    def execute_contentproviders(self):
+    def get_contentproviders(self):
+        cmd = [self.command, "run_base_contentproviders.R", self.wd]
 
     def run(self):
         while True:

From 96aa190317e883291c99405a66df58aa21da8a39 Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Mon, 4 Apr 2022 19:24:14 +0200
Subject: [PATCH 04/13] api cleanup; contenprovider wip

---
 docker-compose.yml                            |  30 ++--
 server/workers/api/src/apis/base.py           |  49 ++++++-
 server/workers/api/src/app.py                 |  41 +++++-
 server/workers/api/src/utils/__init__.py      |   0
 server/workers/api/src/utils/monkeypatches.py | 135 ------------------
 server/workers/base/src/base.py               |  28 +++-
 server/workers/persistence/src/app.py         |   5 +-
 7 files changed, 127 insertions(+), 161 deletions(-)
 delete mode 100644 server/workers/api/src/utils/__init__.py
 delete mode 100644 server/workers/api/src/utils/monkeypatches.py

diff --git a/docker-compose.yml b/docker-compose.yml
index caaa1b230..4c5c21640 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,10 +11,9 @@ services:
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
     command: postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf
     volumes:
-      # - ~/data/OKMaps/${COMPOSE_PROJECT_NAME}/postgresql/data:/var/lib/postgresql/data
       - db_data:/var/lib/postgresql/data
-      - ./server/workers/pg_hba.conf:/etc/pg_hba.conf
-      - ./server/workers/postgresql.conf:/etc/postgresql.conf
+      - ./pg_hba.conf:/etc/pg_hba.conf
+      - ./postgresql.conf:/etc/postgresql.conf
     networks: 
       - headstart
 
@@ -35,8 +34,7 @@ services:
     command: ["redis-server", "/etc/redis/redis.conf", "--bind", "${REDIS_HOST}", "--appendonly", "yes", "--port", "${REDIS_PORT}"]
     volumes:
         - 'redis:/var/lib/redis/data'
-        - ./server/workers/redis.conf:/etc/redis/redis.conf
-        - ./server/workers/certs:/etc/certs
+        - ./redis.conf:/etc/redis/redis.conf
     ports:
      - "127.0.0.1:${REDIS_PORT}:6379"
     restart: always
@@ -57,6 +55,8 @@ services:
       DATABASES: "${DATABASES}"
       FLASK_ENV: "${FLASK_ENV}"
     command: ["gunicorn",  "--workers", "10", "--threads", "2", "-b", "0.0.0.0:${API_PORT}", "app:app", "--timeout", "300"]
+    volumes:
+      - ./api_cache:/var/api_cache
     depends_on:
       - redis
     networks: 
@@ -81,8 +81,6 @@ services:
 
   triple:
     image: triple:${SERVICE_VERSION}
-    env_file:
-      - server/workers/triple/triple.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
@@ -105,8 +103,6 @@ services:
 
   gsheets:
     image: gsheets:${SERVICE_VERSION}
-    env_file:
-      - server/workers/gsheets/gsheets.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
@@ -122,8 +118,6 @@ services:
 
   dataprocessing:
     image: dataprocessing:${SERVICE_VERSION}
-    env_file:
-    - server/workers/dataprocessing/dataprocessing.env
     environment:
       SERVICE_VERSION: "${SERVICE_VERSION}"
       REDIS_HOST: "${REDIS_HOST}"
@@ -131,6 +125,12 @@ services:
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
+      LOGFILE: "${LOGFILE}"
+      RENV_VERSION: 0.14.0-5
+      CRAN_REPOS: https://cran.wu.ac.at
+      LC_ALL: "en_US.UTF-8"
+      LANG: "en_US.UTF-8"
+      RENV_PATHS_CACHE: /renv/cache
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -174,13 +174,12 @@ services:
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -199,13 +198,12 @@ services:
       REDIS_DB: "${REDIS_DB}"
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
-      LOGFILE: "/var/log/headstart/headstart.log"
+      LOGFILE: "${LOGFILE}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"
       LANG: "en_US.UTF-8"
       RENV_PATHS_CACHE: /renv/cache
-      PYTHONIOENCODING: "utf-8"
     restart: always
     volumes:
       - /opt/local/renv/cache:/renv/cache
@@ -219,6 +217,8 @@ volumes:
   redis:
   db_data:
     driver: local
+  api_cache:
+    driver: local
 
 networks:
   headstart:
\ No newline at end of file
diff --git a/server/workers/api/src/apis/base.py b/server/workers/api/src/apis/base.py
index 95ef293c0..97e6a7f87 100644
--- a/server/workers/api/src/apis/base.py
+++ b/server/workers/api/src/apis/base.py
@@ -85,10 +85,57 @@ def post(self):
             base_ns.logger.error(e)
             abort(500, "Problem encountered, check logs.")
 
+def get_or_create_contentprovider_lookup():
+    try:
+        k = str(uuid.uuid4())
+        d = {"id": k, "params": {},
+                "endpoint": "contentproviders"}
+        base_ns.logger.debug(d)
+        redis_store.rpush("base", json.dumps(d))
+        result = get_key(redis_store, k)
+        df = pd.DataFrame(json.loads(result))
+        df.set_index("internal_name", inplace=True)
+        cp_dict = df.name.to_dict()
+        return cp_dict
+    except Exception as e:
+        base_ns.logger.error(e)
+   
+contentprovider_lookup = get_or_create_contentprovider_lookup()
+base_ns.logger.debug(len(contentprovider_lookup))
+
+@base_ns.route('contentprovider')
+class ContentProvider(Resource):
+    @base_ns.doc(responses={200: 'OK',
+                              400: 'Invalid search parameters'})
+    @base_ns.produces(["application/json"])
+    def post(self):
+        """
+        params: can be empty
+        content_provider: BASE internal name, e.g. "fthsaugsburg"
+
+        returns: json
+        {"contentprovider_short": "fthsaugsburg",
+         "contentprovider_long": "OPUS - Publikationsserver der Hochschule Augsburg"}
+        """
+        params = request.get_json()
+        if not params:
+            result = contentprovider_lookup
+        else:
+            result = contentprovider_lookup.get(params["repo"])
+        try:
+            headers = {}
+            headers["Content-Type"] = "application/json"
+            return make_response(result,
+                                 200,
+                                 headers)
+        except Exception as e:
+            base_ns.logger.error(e)
+            abort(500, "Problem encountered, check logs.")
+        
 
 
 @base_ns.route('/service_version')
 class ServiceVersion(Resource):
     def get(self):
         result = {"service_version": os.getenv("SERVICE_VERSION")}
-        return make_response(result, 200, {"Content-Type": "application/json"})
\ No newline at end of file
+        return make_response(result, 200, {"Content-Type": "application/json"})
diff --git a/server/workers/api/src/app.py b/server/workers/api/src/app.py
index 5226688b0..8c080e8f6 100644
--- a/server/workers/api/src/app.py
+++ b/server/workers/api/src/app.py
@@ -1,9 +1,10 @@
 import os
 import sys
-from flask import Flask, redirect, url_for
+from flask import Flask
 from flask_restx import Api
 from flask_cors import CORS
 from werkzeug.middleware.proxy_fix import ProxyFix
+import logging
 
 from apis.triple import triple_ns
 from apis.gsheets import gsheets_ns
@@ -12,19 +13,45 @@
 from apis.openaire import openaire_ns
 from apis.create_vis import vis_ns
 
-from utils.monkeypatches import ReverseProxied, __schema__, specs_url, _register_apidoc
-import logging
 
+class ReverseProxied(object):
+    '''Wrap the application in this middleware and configure the
+    front-end server to add these headers, to let you quietly bind
+    this to a URL other than / and to an HTTP scheme that is
+    different than what is used locally.
+
+    location /myprefix {
+        proxy_pass http://192.168.0.1:5001;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Script-Name /myprefix;
+        }
+
+    :param app: the WSGI application
+    '''
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
+        if script_name:
+            environ['SCRIPT_NAME'] = script_name
+            path_info = environ['PATH_INFO']
+            if path_info.startswith(script_name):
+                environ['PATH_INFO'] = path_info[len(script_name):]
+
+        scheme = environ.get('HTTP_X_SCHEME', '')
+        if scheme:
+            environ['wsgi.url_scheme'] = scheme
+        return self.app(environ, start_response)
 
 def api_patches(app):
-    Api._register_apidoc = _register_apidoc
-    Api.__schema__ = __schema__
-    Api.specs_url = specs_url
 
     api_fixed = Api(
         app,
         title="Head Start API",
-        description="Head Start API demo",
+        description="Head Start API",
         version="0.1",
         prefix='/api',
         doc="/docs")
diff --git a/server/workers/api/src/utils/__init__.py b/server/workers/api/src/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/server/workers/api/src/utils/monkeypatches.py b/server/workers/api/src/utils/monkeypatches.py
deleted file mode 100644
index 4d2edecd5..000000000
--- a/server/workers/api/src/utils/monkeypatches.py
+++ /dev/null
@@ -1,135 +0,0 @@
-import os
-import json
-import yaml
-import logging
-from flask import url_for
-from flask_restx import apidoc
-from flask_restx.swagger import Swagger
-from werkzeug.utils import cached_property
-
-log = logging.getLogger(__name__)
-
-
-class ReverseProxied(object):
-    '''Wrap the application in this middleware and configure the
-    front-end server to add these headers, to let you quietly bind
-    this to a URL other than / and to an HTTP scheme that is
-    different than what is used locally.
-
-    location /myprefix {
-        proxy_pass http://192.168.0.1:5001;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Scheme $scheme;
-        proxy_set_header X-Script-Name /myprefix;
-        }
-
-    :param app: the WSGI application
-    '''
-    def __init__(self, app):
-        self.app = app
-
-    def __call__(self, environ, start_response):
-        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
-        if script_name:
-            environ['SCRIPT_NAME'] = script_name
-            path_info = environ['PATH_INFO']
-            if path_info.startswith(script_name):
-                environ['PATH_INFO'] = path_info[len(script_name):]
-
-        scheme = environ.get('HTTP_X_SCHEME', '')
-        if scheme:
-            environ['wsgi.url_scheme'] = scheme
-        return self.app(environ, start_response)
-
-
-# from https://github.com/noirbizarre/flask-restplus/issues/517
-def _register_apidoc(self, app):
-    conf = app.extensions.setdefault('restx', {})
-    custom_apidoc = apidoc.Apidoc('restx_doc', 'flask_restx.apidoc',
-                                  template_folder='templates',
-                                  static_folder='static',
-                                  static_url_path="/swaggerui")
-
-    @custom_apidoc.add_app_template_global
-    def swagger_static(filename):
-        return url_for('restx_doc.static', filename=filename)
-
-    if not conf.get('apidoc_registered', False):
-        app.register_blueprint(custom_apidoc)
-    conf['apidoc_registered'] = True
-
-
-# from https://github.com/noirbizarre/flask-restplus/pull/596/files
-# make swagger work behind reverse proxy
-
-@cached_property
-def __schema__(self):
-    '''
-    The Swagger specifications/schema for this API
-    :returns dict: the schema as a serializable dict
-    '''
-    if not self._schema:
-        try:
-            self._schema = Swagger(self).as_dict()
-            if self.behind_proxy and "host" in self._schema:
-                del self._schema["host"]
-        except Exception:
-            # Log the source exception for debugging purpose
-            # and return an error message
-            msg = 'Unable to render schema'
-            log.exception(msg)  # This will provide a full traceback
-            return {'error': msg}
-    return self._schema
-
-
-@property
-def specs_url(self):
-    '''
-    The Swagger specifications absolute url (ie. `swagger.json`)
-    Use a relative url when behind a proxy.
-    :rtype: str
-    '''
-    if self.behind_proxy:
-        # Use relative URL.
-        external = False
-    else:
-        external = True
-    url = url_for(self.endpoint('specs'), _external=external)
-    # from https://github.com/noirbizarre/flask-restplus/pull/226/files
-    if self.app.config.get('SWAGGER_BASEPATH', ''):
-        prefix = url.split('/swagger.json')[0]
-        url = prefix + self.app.config.get('SWAGGER_BASEPATH', '') + '/swagger.json'
-        return url
-    return url
-
-
-def inject_flasgger(app):
-    from flasgger import Swagger
-    with open("config/swagger.json") as infile:
-        specs = json.load(infile)
-        swagger = yaml.load(json.dumps(specs))
-    swagger["host"] = os.getenv("HOST_IP", "localhost:5001")
-    if swagger["host"] == "localhost:5001":
-        swagger["schemes"] = ["http"]
-    Swagger(app, template=swagger, config=getSwaggerConfig())
-    return app
-
-
-def getSwaggerConfig():
-    return {
-        "headers": [
-        ],
-        "specs": [
-            {
-                "endpoint": 'apispec',
-                "route": '/apispec.json',
-                "rule_filter": lambda rule: True,  # all in
-                "model_filter": lambda tag: True,  # all in
-            }
-        ],
-        "static_url_path": "/flasgger_static",
-        "static_folder": "static",  # must be set by user
-        "swagger_ui": True,
-        "specs_route": "/"
-    }
diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py
index a02d87f1b..6d6f38898 100644
--- a/server/workers/base/src/base.py
+++ b/server/workers/base/src/base.py
@@ -54,6 +54,23 @@ def execute_search(self, params):
 
     def get_contentproviders(self):
         cmd = [self.command, "run_base_contentproviders.R", self.wd]
+        try:
+            proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                    encoding='utf-8')
+            stdout, stderr = proc.communicate(json.dumps({}))
+            output = [o for o in stdout.split('\n') if len(o) > 0]
+            error = [o for o in stderr.split('\n') if len(o) > 0]
+            raw = json.loads(output[-1])
+            if isinstance(raw, dict) and raw.get('status') == "error":
+                res = raw
+            else:
+                contentproviders = pd.DataFrame(raw)
+                res = {}
+                res["contentproviders"] = contentproviders
+            return res
+        except Exception as e:
+            self.logger.error(e)
+            self.logger.error(error)
 
     def run(self):
         while True:
@@ -74,4 +91,13 @@ def run(self):
 
             if endpoint == "contentproviders":
                 try:
-                    res = self.execute
+                    res = self.get_contentproviders()
+                    res["id"] = k
+                    if res.get("status") == "error" or params.get('raw') is True:
+                        self.redis_store.set(k+"_output", json.dumps(res))
+                    else:
+                        self.redis_store.rpush("input_data", json.dumps(res).encode('utf8'))
+                except Exception as e:
+                    self.logger.exception("Exception during retrieval of contentproviders.")
+                    self.logger.error(params)
+
diff --git a/server/workers/persistence/src/app.py b/server/workers/persistence/src/app.py
index 9ba0d3fcd..6c155300d 100644
--- a/server/workers/persistence/src/app.py
+++ b/server/workers/persistence/src/app.py
@@ -4,9 +4,9 @@
 from flask_restx import Api
 from flask_cors import CORS
 from werkzeug.middleware.proxy_fix import ProxyFix
+import logging
 
 from apis.persistence import persistence_ns
-import logging
 
 class ReverseProxied(object):
     '''Wrap the application in this middleware and configure the
@@ -45,7 +45,7 @@ def api_patches(app):
     api_fixed = Api(
         app,
         title="Head Start API",
-        description="Head Start API demo",
+        description="Head Start API",
         version="0.1",
         prefix='/api',
         doc="/docs")
@@ -63,6 +63,7 @@ def api_patches(app):
 
 api = api_patches(app)
 api.add_namespace(persistence_ns, path='/persistence')
+
 app.logger.debug(app.config)
 app.logger.debug(app.url_map)
 

From fd777778df18db769bdc5b73b7dd9d1a28abe796 Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Mon, 4 Apr 2022 22:37:20 +0200
Subject: [PATCH 05/13] Rclient and Rwrapper

---
 .../other-scripts/run_base_contentproviders.R       |  1 +
 server/workers/base/src/base.py                     | 13 ++++++-------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/server/preprocessing/other-scripts/run_base_contentproviders.R b/server/preprocessing/other-scripts/run_base_contentproviders.R
index d0eaa6507..c51b5c791 100644
--- a/server/preprocessing/other-scripts/run_base_contentproviders.R
+++ b/server/preprocessing/other-scripts/run_base_contentproviders.R
@@ -9,6 +9,7 @@ renv::restore(lockfile = '../renv.lock')
 Sys.setlocale(category="LC_ALL", locale = "en_US.UTF-8")
 
 library(jsonlite)
+library(rbace)
 library(logging)
 
 source('utils.R')
diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py
index 6d6f38898..ab1d559f8 100644
--- a/server/workers/base/src/base.py
+++ b/server/workers/base/src/base.py
@@ -1,3 +1,4 @@
+import os
 import json
 import subprocess
 import pandas as pd
@@ -53,11 +54,12 @@ def execute_search(self, params):
             raise
 
     def get_contentproviders(self):
-        cmd = [self.command, "run_base_contentproviders.R", self.wd]
+        runner = os.path.abspath(os.path.join(self.wd, "run_base_contentproviders.R"))
+        cmd = [self.command, runner, self.wd]
         try:
             proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                     encoding='utf-8')
-            stdout, stderr = proc.communicate(json.dumps({}))
+            stdout, stderr = proc.communicate()
             output = [o for o in stdout.split('\n') if len(o) > 0]
             error = [o for o in stderr.split('\n') if len(o) > 0]
             raw = json.loads(output[-1])
@@ -66,7 +68,7 @@ def get_contentproviders(self):
             else:
                 contentproviders = pd.DataFrame(raw)
                 res = {}
-                res["contentproviders"] = contentproviders
+                res["contentproviders"] = contentproviders.to_json(orient='records')
             return res
         except Exception as e:
             self.logger.error(e)
@@ -93,10 +95,7 @@ def run(self):
                 try:
                     res = self.get_contentproviders()
                     res["id"] = k
-                    if res.get("status") == "error" or params.get('raw') is True:
-                        self.redis_store.set(k+"_output", json.dumps(res))
-                    else:
-                        self.redis_store.rpush("input_data", json.dumps(res).encode('utf8'))
+                    self.redis_store.set(k+"_output", json.dumps(res))
                 except Exception as e:
                     self.logger.exception("Exception during retrieval of contentproviders.")
                     self.logger.error(params)

From edcc99298df981880adac67267287975a7969eff Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Tue, 5 Apr 2022 00:57:49 +0200
Subject: [PATCH 06/13] contentprovider lookup should be working

---
 server/workers/api/src/apis/base.py | 41 +++++++++++++++--------------
 server/workers/base/src/base.py     |  2 ++
 2 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/server/workers/api/src/apis/base.py b/server/workers/api/src/apis/base.py
index 97e6a7f87..ccde97522 100644
--- a/server/workers/api/src/apis/base.py
+++ b/server/workers/api/src/apis/base.py
@@ -39,6 +39,22 @@
                                                       description='raw results from ElasticSearch')})
 
 
+def get_or_create_contentprovider_lookup():
+    try:
+        k = str(uuid.uuid4())
+        d = {"id": k, "params": {},"endpoint": "contentproviders"}
+        base_ns.logger.debug(d)
+        redis_store.rpush("base", json.dumps(d))
+        result = get_key(redis_store, k)
+        df = pd.DataFrame(json.loads(result["contentproviders"]))
+        df.set_index("internal_name", inplace=True)
+        cp_dict = df.name.to_dict()
+        return cp_dict
+    except Exception as e:
+        base_ns.logger.error(e)
+   
+contentprovider_lookup = get_or_create_contentprovider_lookup()
+
 @base_ns.route('/search')
 class Search(Resource):
     @base_ns.doc(responses={200: 'OK',
@@ -53,6 +69,9 @@ def post(self):
         if "optradio" in params:
             del params["optradio"]
         errors = search_param_schema.validate(params, partial=True)
+        if "repo" in params:
+            contentprovider_long = contentprovider_lookup.get(params["repo"])
+            params["contentprovider_long"] = contentprovider_long
         params["limit"] = 120
         params["list_size"] = 100
         base_ns.logger.debug(errors)
@@ -85,25 +104,7 @@ def post(self):
             base_ns.logger.error(e)
             abort(500, "Problem encountered, check logs.")
 
-def get_or_create_contentprovider_lookup():
-    try:
-        k = str(uuid.uuid4())
-        d = {"id": k, "params": {},
-                "endpoint": "contentproviders"}
-        base_ns.logger.debug(d)
-        redis_store.rpush("base", json.dumps(d))
-        result = get_key(redis_store, k)
-        df = pd.DataFrame(json.loads(result))
-        df.set_index("internal_name", inplace=True)
-        cp_dict = df.name.to_dict()
-        return cp_dict
-    except Exception as e:
-        base_ns.logger.error(e)
-   
-contentprovider_lookup = get_or_create_contentprovider_lookup()
-base_ns.logger.debug(len(contentprovider_lookup))
-
-@base_ns.route('contentprovider')
+@base_ns.route('/contentproviders')
 class ContentProvider(Resource):
     @base_ns.doc(responses={200: 'OK',
                               400: 'Invalid search parameters'})
@@ -121,7 +122,7 @@ def post(self):
         if not params:
             result = contentprovider_lookup
         else:
-            result = contentprovider_lookup.get(params["repo"])
+            result = {"contentprovider_long": contentprovider_lookup.get(params["repo"])}
         try:
             headers = {}
             headers["Content-Type"] = "application/json"
diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py
index ab1d559f8..bcd671f6a 100644
--- a/server/workers/base/src/base.py
+++ b/server/workers/base/src/base.py
@@ -90,6 +90,7 @@ def run(self):
                 except Exception as e:
                     self.logger.exception("Exception during data retrieval.")
                     self.logger.error(params)
+                    self.logger.error(e)
 
             if endpoint == "contentproviders":
                 try:
@@ -99,4 +100,5 @@ def run(self):
                 except Exception as e:
                     self.logger.exception("Exception during retrieval of contentproviders.")
                     self.logger.error(params)
+                    self.logger.error(e)
 

From 504fd8c9f2bbeaf41ac944d9b1cfe18a23217e32 Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Tue, 5 Apr 2022 14:24:02 +0200
Subject: [PATCH 07/13] added fields to validator

---
 server/workers/api/src/apis/request_validators.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/server/workers/api/src/apis/request_validators.py b/server/workers/api/src/apis/request_validators.py
index 8a85f4666..d55b69be3 100644
--- a/server/workers/api/src/apis/request_validators.py
+++ b/server/workers/api/src/apis/request_validators.py
@@ -21,6 +21,8 @@ class SearchParamSchema(Schema):
     unique_id = fields.Str()
     raw = fields.Boolean()
     sg_method = fields.Str()
+    repo = fields.Str()
+    coll = fields.Str()
     vis_id = fields.Str(default=None)
     optradio = fields.Str()
     service = fields.Str()

From b27d6bf8eed789ff4f3da4271e95041be3d9164e Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Tue, 5 Apr 2022 17:21:48 +0200
Subject: [PATCH 08/13] fixed integration with persistence

---
 server/services/search.php                        | 9 +++++++++
 server/workers/api/src/apis/base.py               | 7 ++++---
 server/workers/api/src/apis/request_validators.py | 1 +
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/server/services/search.php b/server/services/search.php
index ef2e282da..1de7d3145 100644
--- a/server/services/search.php
+++ b/server/services/search.php
@@ -109,6 +109,15 @@ function search($service_integration, $dirty_query
     }
     $unique_id = ($precomputed_id === null)?($unique_id):($precomputed_id);
     $post_params["vis_id"] = $unique_id;
+    if (array_key_exists("repo", $post_params)) {
+      $payload = json_encode(array("repo" => $post_params["repo"]));
+      $res = $apiclient->call_api($endpoint . "/contentproviders", $payload);
+      $res = $res["result"];
+      $res = json_decode($res, true);
+      $contentprovider_long = $res["contentprovider_long"];
+      $post_params["contentprovider_long"] = $contentprovider_long;
+      $param_types[] = "contentprovider_long";
+    }
     $params_json = packParamsJSON($param_types, $post_params);
 
     if($retrieve_cached_map) {
diff --git a/server/workers/api/src/apis/base.py b/server/workers/api/src/apis/base.py
index ccde97522..e95a9393f 100644
--- a/server/workers/api/src/apis/base.py
+++ b/server/workers/api/src/apis/base.py
@@ -112,13 +112,14 @@ class ContentProvider(Resource):
     def post(self):
         """
         params: can be empty
-        content_provider: BASE internal name, e.g. "fthsaugsburg"
+        content_provider: BASE internal name, e.g. "ftunivlausanne"
 
         returns: json
-        {"contentprovider_short": "fthsaugsburg",
-         "contentprovider_long": "OPUS - Publikationsserver der Hochschule Augsburg"}
+        {"contentprovider_short": "ftunivlausanne",
+         "contentprovider_long": "Université de Lausanne (UNIL): Serval - Serveur académique lausannois"}
         """
         params = request.get_json()
+        base_ns.logger.debug(params)
         if not params:
             result = contentprovider_lookup
         else:
diff --git a/server/workers/api/src/apis/request_validators.py b/server/workers/api/src/apis/request_validators.py
index d55b69be3..1ab626ff1 100644
--- a/server/workers/api/src/apis/request_validators.py
+++ b/server/workers/api/src/apis/request_validators.py
@@ -22,6 +22,7 @@ class SearchParamSchema(Schema):
     raw = fields.Boolean()
     sg_method = fields.Str()
     repo = fields.Str()
+    contentprovider_long = fields.Str()
     coll = fields.Str()
     vis_id = fields.Str(default=None)
     optradio = fields.Str()

From 834a97f4c592a08fc2f8f71471389fcf6eb6d7a3 Mon Sep 17 00:00:00 2001
From: chreman <web@christopherkittel.eu>
Date: Tue, 3 May 2022 10:53:12 +0200
Subject: [PATCH 09/13] rename contentprovider param

---
 server/services/search.php                        | 6 +++---
 server/workers/api/src/apis/base.py               | 8 ++++----
 server/workers/api/src/apis/request_validators.py | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/server/services/search.php b/server/services/search.php
index 1de7d3145..f34a2fd8a 100644
--- a/server/services/search.php
+++ b/server/services/search.php
@@ -114,9 +114,9 @@ function search($service_integration, $dirty_query
       $res = $apiclient->call_api($endpoint . "/contentproviders", $payload);
       $res = $res["result"];
       $res = json_decode($res, true);
-      $contentprovider_long = $res["contentprovider_long"];
-      $post_params["contentprovider_long"] = $contentprovider_long;
-      $param_types[] = "contentprovider_long";
+      $repo_name = $res["repo_name"];
+      $post_params["repo_name"] = $repo_name;
+      $param_types[] = "repo_name";
     }
     $params_json = packParamsJSON($param_types, $post_params);
 
diff --git a/server/workers/api/src/apis/base.py b/server/workers/api/src/apis/base.py
index e95a9393f..06625307d 100644
--- a/server/workers/api/src/apis/base.py
+++ b/server/workers/api/src/apis/base.py
@@ -70,8 +70,8 @@ def post(self):
             del params["optradio"]
         errors = search_param_schema.validate(params, partial=True)
         if "repo" in params:
-            contentprovider_long = contentprovider_lookup.get(params["repo"])
-            params["contentprovider_long"] = contentprovider_long
+            repo_name = contentprovider_lookup.get(params["repo"])
+            params["repo_name"] = repo_name
         params["limit"] = 120
         params["list_size"] = 100
         base_ns.logger.debug(errors)
@@ -116,14 +116,14 @@ def post(self):
 
         returns: json
         {"contentprovider_short": "ftunivlausanne",
-         "contentprovider_long": "Université de Lausanne (UNIL): Serval - Serveur académique lausannois"}
+         "repo_name": "Université de Lausanne (UNIL): Serval - Serveur académique lausannois"}
         """
         params = request.get_json()
         base_ns.logger.debug(params)
         if not params:
             result = contentprovider_lookup
         else:
-            result = {"contentprovider_long": contentprovider_lookup.get(params["repo"])}
+            result = {"repo_name": contentprovider_lookup.get(params["repo"])}
         try:
             headers = {}
             headers["Content-Type"] = "application/json"
diff --git a/server/workers/api/src/apis/request_validators.py b/server/workers/api/src/apis/request_validators.py
index 1ab626ff1..0f3e7a669 100644
--- a/server/workers/api/src/apis/request_validators.py
+++ b/server/workers/api/src/apis/request_validators.py
@@ -22,7 +22,7 @@ class SearchParamSchema(Schema):
     raw = fields.Boolean()
     sg_method = fields.Str()
     repo = fields.Str()
-    contentprovider_long = fields.Str()
+    repo_name = fields.Str()
     coll = fields.Str()
     vis_id = fields.Str(default=None)
     optradio = fields.Str()

From f4dbd6c2ab58b454df0d56073efc67e9c826cd36 Mon Sep 17 00:00:00 2001
From: Jan Konstant <konstantjan@seznam.cz>
Date: Tue, 3 May 2022 15:20:03 +0200
Subject: [PATCH 10/13] added content provider info into Headstart

---
 vis/js/components/ContextLine.js              |  4 ++-
 vis/js/reducers/contextLine.js                |  1 +
 .../templates/contextfeatures/DataSource.jsx  | 29 +++++++++++++++++--
 .../infomodal/subcomponents/DataSource.jsx    | 16 ++++++++--
 .../subcomponents/StandardKMInfo.jsx          |  3 +-
 .../subcomponents/StandardSGInfo.jsx          |  3 +-
 6 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/vis/js/components/ContextLine.js b/vis/js/components/ContextLine.js
index 4c90a3858..d7e1e75fe 100644
--- a/vis/js/components/ContextLine.js
+++ b/vis/js/components/ContextLine.js
@@ -53,8 +53,10 @@ class ContextLine extends React.Component {
         </NumArticles>
         {defined(params.dataSource) && (
           <DataSource
-            value={params.dataSource}
             label={localization.source_label}
+            source={params.dataSource}
+            contentProvider={params.contentProvider}
+            popoverContainer={this.props.popoverContainer}
           />
         )}
         {defined(params.timespan) && <Timespan>{params.timespan}</Timespan>}
diff --git a/vis/js/reducers/contextLine.js b/vis/js/reducers/contextLine.js
index 8a1a5de13..262e3e349 100644
--- a/vis/js/reducers/contextLine.js
+++ b/vis/js/reducers/contextLine.js
@@ -38,6 +38,7 @@ const contextLine = (state = {}, action) => {
           typeof config.service_name !== "undefined"
             ? config.service_name
             : config.service_names[context.service],
+        contentProvider: context.params ? context.params.repo_name : null,
         paperCount:
           config.create_title_from_context_style === "viper"
             ? papers.filter((p) => p.resulttype.includes("publication")).length
diff --git a/vis/js/templates/contextfeatures/DataSource.jsx b/vis/js/templates/contextfeatures/DataSource.jsx
index b4a7d362b..57dff4e17 100644
--- a/vis/js/templates/contextfeatures/DataSource.jsx
+++ b/vis/js/templates/contextfeatures/DataSource.jsx
@@ -1,13 +1,38 @@
 import React from "react";
+import { shorten } from "../../utils/string";
+import HoverPopover from "../HoverPopover";
+
+const MAX_CONTENT_PROVIDER_LENGTH = 20;
+
+const DataSource = ({ label, source, contentProvider, popoverContainer }) => {
+  if (contentProvider) {
+    const content = (
+      shorten(contentProvider, MAX_CONTENT_PROVIDER_LENGTH) + ` via ${source}`
+    ).replace("... via ", "...via ");
+
+    return (
+      // html template starts here
+      <span id="source" className="context_item">
+        {label}:{" "}
+        <HoverPopover
+          id="doctypes-popover"
+          container={popoverContainer}
+          content={`${contentProvider} (via ${source})`}
+        >
+          <span className="context_moreinfo">{content}</span>
+        </HoverPopover>
+      </span>
+      // html template ends here
+    );
+  }
 
-const DataSource = ({ label, value }) => {
   return (
     // html template starts here
     <span
       id="source"
       className="context_item"
       dangerouslySetInnerHTML={{
-        __html: `${label}: ${value}`,
+        __html: `${label}: ${source}`,
       }}
     ></span>
     // html template ends here
diff --git a/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx b/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx
index 5de5c5141..b9a72ca0a 100644
--- a/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx
+++ b/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx
@@ -1,12 +1,24 @@
 import React from "react";
 
-const DataSource = ({ source, description, logo }) => {
+const DataSource = ({ source, contentProvider, description, logo }) => {
+  const getDataSource = () => {
+    if (!contentProvider) {
+      return <strong>{source}</strong>;
+    }
+
+    return (
+      <>
+        {contentProvider} (via {source})
+      </>
+    );
+  };
+
   return (
     // html template starts here
     <>
       <h3>Data source</h3>
       <p>
-        The data is taken from <strong>{source}</strong>. {description}
+        The data is taken from {getDataSource()}. {description}
       </p>
       {!!logo && <p className="datasource-logo">{logo}</p>}
     </>
diff --git a/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx b/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx
index 61f82721a..2fb4e1b2c 100644
--- a/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx
+++ b/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx
@@ -8,7 +8,7 @@ const StandardKMInfo = ({
   serviceName,
   serviceDesc,
   serviceLogo,
-  params: { query, customTitle },
+  params: { query, customTitle, repo_name },
 }) => {
   return (
     // html template starts here
@@ -72,6 +72,7 @@ const StandardKMInfo = ({
         {!!serviceName && (
           <DataSource
             source={serviceName}
+            contentProvider={repo_name}
             description={serviceDesc}
             logo={serviceLogo}
           />
diff --git a/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx b/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx
index 716781f61..ed2e82d38 100644
--- a/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx
+++ b/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx
@@ -8,7 +8,7 @@ const StandardSGInfo = ({
   serviceName,
   serviceDesc,
   serviceLogo,
-  params: { query, customTitle },
+  params: { query, customTitle, repo_name },
 }) => {
   return (
     // html template starts here
@@ -44,6 +44,7 @@ const StandardSGInfo = ({
         {!!serviceName && (
           <DataSource
             source={serviceName}
+            contentProvider={repo_name}
             description={serviceDesc}
             logo={serviceLogo}
           />

From b44ad3abf62a1f0151e1f9e31625adfb00878fd9 Mon Sep 17 00:00:00 2001
From: Jan Konstant <konstantjan@seznam.cz>
Date: Mon, 9 May 2022 09:03:06 +0200
Subject: [PATCH 11/13] changed the repo name lenght to 6 chars

---
 vis/js/templates/contextfeatures/DataSource.jsx | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/vis/js/templates/contextfeatures/DataSource.jsx b/vis/js/templates/contextfeatures/DataSource.jsx
index 57dff4e17..9558c52c3 100644
--- a/vis/js/templates/contextfeatures/DataSource.jsx
+++ b/vis/js/templates/contextfeatures/DataSource.jsx
@@ -2,13 +2,11 @@ import React from "react";
 import { shorten } from "../../utils/string";
 import HoverPopover from "../HoverPopover";
 
-const MAX_CONTENT_PROVIDER_LENGTH = 20;
+const MAX_CONTENT_PROVIDER_LENGTH = 6;
 
 const DataSource = ({ label, source, contentProvider, popoverContainer }) => {
   if (contentProvider) {
-    const content = (
-      shorten(contentProvider, MAX_CONTENT_PROVIDER_LENGTH) + ` via ${source}`
-    ).replace("... via ", "...via ");
+    const content = shorten(contentProvider, MAX_CONTENT_PROVIDER_LENGTH);
 
     return (
       // html template starts here

From 4a7fe64f66ccdcececfd85280240483517d5d6a5 Mon Sep 17 00:00:00 2001
From: Jan Konstant <konstantjan@seznam.cz>
Date: Mon, 9 May 2022 09:22:48 +0200
Subject: [PATCH 12/13] hiding new share buttons in embed mode

---
 vis/js/reducers/modals.js | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/vis/js/reducers/modals.js b/vis/js/reducers/modals.js
index cd7517cba..b4a956b2c 100644
--- a/vis/js/reducers/modals.js
+++ b/vis/js/reducers/modals.js
@@ -44,8 +44,8 @@ const modals = (
         openCitationModal: false,
         citedPaper: null,
         exportedPaper: null,
-        showTwitterButton: !!action.configObject.show_twitter_button,
-        showEmailButton: !!action.configObject.show_email_button,
+        showTwitterButton: showTwitterShare(action.configObject, action.contextObject),
+        showEmailButton: showEmailShare(action.configObject, action.contextObject),
       };
     case "OPEN_EMBED_MODAL":
       return {
@@ -148,3 +148,17 @@ const getSheetID = (config, context) => {
 
   return config.files[0].file;
 };
+
+const showTwitterShare = (config, context) => {
+  if (config.credit_embed) {
+    return false;
+  }
+  return !!config.show_twitter_button;
+};
+
+const showEmailShare = (config, context) => {
+  if (config.credit_embed) {
+    return false;
+  }
+  return !!config.show_email_button;
+};

From c083c0f06a2c4e479385ab5444f808667e0b7de6 Mon Sep 17 00:00:00 2001
From: Jan Konstant <62649432+Konstiman@users.noreply.github.com>
Date: Thu, 19 May 2022 10:46:58 +0200
Subject: [PATCH 13/13] Update CHANGELOG.md

---
 CHANGELOG.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a393c9e3..8400cf44e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,15 @@
 # Changelog
 
+## 2022-05-19
+
+### New features
+
+- The BASE content provider is now displayed in the context line & more info modal.
+
+### Changes
+
+- The new share buttons are now hidden in the embed mode.
+
 ## 2022-05-05
 
 ### New features