diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a393c9e3..8400cf44e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2022-05-19 + +### New features + +- The BASE content provider is now displayed in the context line & more info modal. + +### Changes + +- The new share buttons are now hidden in the embed mode. + ## 2022-05-05 ### New features diff --git a/docker-compose-dataworker.yml b/docker-compose-dataworker.yml index 313ca5295..9ca1e3804 100644 --- a/docker-compose-dataworker.yml +++ b/docker-compose-dataworker.yml @@ -12,7 +12,6 @@ services: REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" restart: always volumes: - /opt/local/renv/cache:/renv/cache diff --git a/docker-compose.yml b/docker-compose.yml index 69425a221..4c5c21640 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,10 +11,9 @@ services: POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" command: postgres -c config_file=/etc/postgresql.conf -c hba_file=/etc/pg_hba.conf volumes: - # - ~/data/OKMaps/${COMPOSE_PROJECT_NAME}/postgresql/data:/var/lib/postgresql/data - db_data:/var/lib/postgresql/data - - ./server/workers/pg_hba.conf:/etc/pg_hba.conf - - ./server/workers/postgresql.conf:/etc/postgresql.conf + - ./pg_hba.conf:/etc/pg_hba.conf + - ./postgresql.conf:/etc/postgresql.conf networks: - headstart @@ -35,8 +34,7 @@ services: command: ["redis-server", "/etc/redis/redis.conf", "--bind", "${REDIS_HOST}", "--appendonly", "yes", "--port", "${REDIS_PORT}"] volumes: - 'redis:/var/lib/redis/data' - - ./server/workers/redis.conf:/etc/redis/redis.conf - - ./server/workers/certs:/etc/certs + - ./redis.conf:/etc/redis/redis.conf ports: - "127.0.0.1:${REDIS_PORT}:6379" restart: always @@ -52,12 +50,13 @@ services: REDIS_PORT: "${REDIS_PORT}" REDIS_PASSWORD: "${REDIS_PASSWORD}" REDIS_DB: "${REDIS_DB}" - REDIS_SSL: "${REDIS_SSL}" BEHIND_PROXY: "${BEHIND_PROXY}" DEFAULT_DATABASE: "${DEFAULT_DATABASE}" DATABASES: "${DATABASES}" FLASK_ENV: "${FLASK_ENV}" command: ["gunicorn", "--workers", "10", "--threads", "2", "-b", "0.0.0.0:${API_PORT}", "app:app", "--timeout", "300"] + volumes: + - ./api_cache:/var/api_cache depends_on: - redis networks: @@ -82,15 +81,12 @@ services: triple: image: triple:${SERVICE_VERSION} - env_file: - - server/workers/triple/triple.env environment: SERVICE_VERSION: "${SERVICE_VERSION}" REDIS_HOST: "${REDIS_HOST}" REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" TRIPLE_USER: "${TRIPLE_USER}" TRIPLE_PASS: "${TRIPLE_PASS}" @@ -107,15 +103,12 @@ services: gsheets: image: gsheets:${SERVICE_VERSION} - env_file: - - server/workers/gsheets/gsheets.env environment: SERVICE_VERSION: "${SERVICE_VERSION}" REDIS_HOST: "${REDIS_HOST}" REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" restart: always depends_on: @@ -125,16 +118,19 @@ services: dataprocessing: image: dataprocessing:${SERVICE_VERSION} - env_file: - - server/workers/dataprocessing/dataprocessing.env environment: SERVICE_VERSION: "${SERVICE_VERSION}" REDIS_HOST: "${REDIS_HOST}" REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" + LOGFILE: "${LOGFILE}" + RENV_VERSION: 0.14.0-5 + CRAN_REPOS: https://cran.wu.ac.at + LC_ALL: "en_US.UTF-8" + LANG: "en_US.UTF-8" + RENV_PATHS_CACHE: /renv/cache restart: always volumes: - /opt/local/renv/cache:/renv/cache @@ -152,7 +148,6 @@ services: REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" LOGFILE: "/var/log/headstart/headstart.log" RENV_VERSION: 0.14.0-5 @@ -178,15 +173,13 @@ services: REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" - LOGFILE: "/var/log/headstart/headstart.log" + LOGFILE: "${LOGFILE}" RENV_VERSION: 0.14.0-5 CRAN_REPOS: https://cran.wu.ac.at LC_ALL: "en_US.UTF-8" LANG: "en_US.UTF-8" RENV_PATHS_CACHE: /renv/cache - PYTHONIOENCODING: "utf-8" restart: always volumes: - /opt/local/renv/cache:/renv/cache @@ -204,15 +197,13 @@ services: REDIS_PORT: "${REDIS_PORT}" REDIS_DB: "${REDIS_DB}" REDIS_PASSWORD: "${REDIS_PASSWORD}" - REDIS_SSL: "${REDIS_SSL}" LOGLEVEL: "${LOGLEVEL}" - LOGFILE: "/var/log/headstart/headstart.log" + LOGFILE: "${LOGFILE}" RENV_VERSION: 0.14.0-5 CRAN_REPOS: https://cran.wu.ac.at LC_ALL: "en_US.UTF-8" LANG: "en_US.UTF-8" RENV_PATHS_CACHE: /renv/cache - PYTHONIOENCODING: "utf-8" restart: always volumes: - /opt/local/renv/cache:/renv/cache @@ -226,6 +217,8 @@ volumes: redis: db_data: driver: local + api_cache: + driver: local networks: headstart: \ No newline at end of file diff --git a/server/preprocessing/other-scripts/run_base_contentproviders.R b/server/preprocessing/other-scripts/run_base_contentproviders.R new file mode 100644 index 000000000..c51b5c791 --- /dev/null +++ b/server/preprocessing/other-scripts/run_base_contentproviders.R @@ -0,0 +1,45 @@ +rm(list = ls()) + +args <- commandArgs(TRUE) +wd <- args[1] +setwd(wd) #Don't forget to set your working directory + +renv::activate() +renv::restore(lockfile = '../renv.lock') +Sys.setlocale(category="LC_ALL", locale = "en_US.UTF-8") + +library(jsonlite) +library(rbace) +library(logging) + +source('utils.R') +if (Sys.getenv("LOGLEVEL") == "DEBUG") { + DEBUG <- FALSE +} else { + DEBUG <- TRUE +} + +if (DEBUG==TRUE){ + setup_logging('DEBUG') +} else { + setup_logging('INFO') +} + + +log <- getLogger('base_repos') + +tryCatch({ + contentproviders <- bs_repositories("") +}, error=function(err){ + log$error(paste("Contentprovider failed", "base", "retrieve_contentproviders", "", err, sep="||")) + failed <- list() + failed$reason <- list(err) + failed$status <- 'error' +}) + + +if (exists('contentproviders')) { + print(toJSON(contentproviders)) +} else { + print(toJSON(failed)) +} diff --git a/server/services/search.php b/server/services/search.php index ef2e282da..f34a2fd8a 100644 --- a/server/services/search.php +++ b/server/services/search.php @@ -109,6 +109,15 @@ function search($service_integration, $dirty_query } $unique_id = ($precomputed_id === null)?($unique_id):($precomputed_id); $post_params["vis_id"] = $unique_id; + if (array_key_exists("repo", $post_params)) { + $payload = json_encode(array("repo" => $post_params["repo"])); + $res = $apiclient->call_api($endpoint . "/contentproviders", $payload); + $res = $res["result"]; + $res = json_decode($res, true); + $repo_name = $res["repo_name"]; + $post_params["repo_name"] = $repo_name; + $param_types[] = "repo_name"; + } $params_json = packParamsJSON($param_types, $post_params); if($retrieve_cached_map) { diff --git a/server/workers/api/src/apis/base.py b/server/workers/api/src/apis/base.py index 95ef293c0..06625307d 100644 --- a/server/workers/api/src/apis/base.py +++ b/server/workers/api/src/apis/base.py @@ -39,6 +39,22 @@ description='raw results from ElasticSearch')}) +def get_or_create_contentprovider_lookup(): + try: + k = str(uuid.uuid4()) + d = {"id": k, "params": {},"endpoint": "contentproviders"} + base_ns.logger.debug(d) + redis_store.rpush("base", json.dumps(d)) + result = get_key(redis_store, k) + df = pd.DataFrame(json.loads(result["contentproviders"])) + df.set_index("internal_name", inplace=True) + cp_dict = df.name.to_dict() + return cp_dict + except Exception as e: + base_ns.logger.error(e) + +contentprovider_lookup = get_or_create_contentprovider_lookup() + @base_ns.route('/search') class Search(Resource): @base_ns.doc(responses={200: 'OK', @@ -53,6 +69,9 @@ def post(self): if "optradio" in params: del params["optradio"] errors = search_param_schema.validate(params, partial=True) + if "repo" in params: + repo_name = contentprovider_lookup.get(params["repo"]) + params["repo_name"] = repo_name params["limit"] = 120 params["list_size"] = 100 base_ns.logger.debug(errors) @@ -85,10 +104,40 @@ def post(self): base_ns.logger.error(e) abort(500, "Problem encountered, check logs.") +@base_ns.route('/contentproviders') +class ContentProvider(Resource): + @base_ns.doc(responses={200: 'OK', + 400: 'Invalid search parameters'}) + @base_ns.produces(["application/json"]) + def post(self): + """ + params: can be empty + content_provider: BASE internal name, e.g. "ftunivlausanne" + + returns: json + {"contentprovider_short": "ftunivlausanne", + "repo_name": "Université de Lausanne (UNIL): Serval - Serveur académique lausannois"} + """ + params = request.get_json() + base_ns.logger.debug(params) + if not params: + result = contentprovider_lookup + else: + result = {"repo_name": contentprovider_lookup.get(params["repo"])} + try: + headers = {} + headers["Content-Type"] = "application/json" + return make_response(result, + 200, + headers) + except Exception as e: + base_ns.logger.error(e) + abort(500, "Problem encountered, check logs.") + @base_ns.route('/service_version') class ServiceVersion(Resource): def get(self): result = {"service_version": os.getenv("SERVICE_VERSION")} - return make_response(result, 200, {"Content-Type": "application/json"}) \ No newline at end of file + return make_response(result, 200, {"Content-Type": "application/json"}) diff --git a/server/workers/api/src/apis/request_validators.py b/server/workers/api/src/apis/request_validators.py index 6833f3080..9c71d84e4 100644 --- a/server/workers/api/src/apis/request_validators.py +++ b/server/workers/api/src/apis/request_validators.py @@ -21,6 +21,9 @@ class SearchParamSchema(Schema): unique_id = fields.Str() raw = fields.Boolean() sg_method = fields.Str() + repo = fields.Str() + repo_name = fields.Str() + coll = fields.Str() vis_id = fields.Str(default=None) optradio = fields.Str() service = fields.Str() diff --git a/server/workers/api/src/apis/utils.py b/server/workers/api/src/apis/utils.py index 05b7bcd8e..bfc4167b6 100644 --- a/server/workers/api/src/apis/utils.py +++ b/server/workers/api/src/apis/utils.py @@ -11,8 +11,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "api", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "api" } redis_store = redis.StrictRedis(**redis_config) diff --git a/server/workers/api/src/app.py b/server/workers/api/src/app.py index 1bb1620ec..f37b75945 100644 --- a/server/workers/api/src/app.py +++ b/server/workers/api/src/app.py @@ -1,9 +1,10 @@ import os import sys -from flask import Flask, redirect, url_for +from flask import Flask from flask_restx import Api from flask_cors import CORS from werkzeug.middleware.proxy_fix import ProxyFix +import logging from apis.triple import triple_ns from apis.gsheets import gsheets_ns @@ -13,19 +14,45 @@ from apis.create_vis import vis_ns from apis.export import export_ns -from utils.monkeypatches import ReverseProxied, __schema__, specs_url, _register_apidoc -import logging +class ReverseProxied(object): + '''Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + location /myprefix { + proxy_pass http://192.168.0.1:5001; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + :param app: the WSGI application + ''' + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + script_name = environ.get('HTTP_X_SCRIPT_NAME', '') + if script_name: + environ['SCRIPT_NAME'] = script_name + path_info = environ['PATH_INFO'] + if path_info.startswith(script_name): + environ['PATH_INFO'] = path_info[len(script_name):] + + scheme = environ.get('HTTP_X_SCHEME', '') + if scheme: + environ['wsgi.url_scheme'] = scheme + return self.app(environ, start_response) def api_patches(app): - Api._register_apidoc = _register_apidoc - Api.__schema__ = __schema__ - Api.specs_url = specs_url api_fixed = Api( app, title="Head Start API", - description="Head Start API demo", + description="Head Start API", version="0.1", prefix='/api', doc="/docs") diff --git a/server/workers/api/src/utils/__init__.py b/server/workers/api/src/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/server/workers/api/src/utils/monkeypatches.py b/server/workers/api/src/utils/monkeypatches.py deleted file mode 100644 index 4d2edecd5..000000000 --- a/server/workers/api/src/utils/monkeypatches.py +++ /dev/null @@ -1,135 +0,0 @@ -import os -import json -import yaml -import logging -from flask import url_for -from flask_restx import apidoc -from flask_restx.swagger import Swagger -from werkzeug.utils import cached_property - -log = logging.getLogger(__name__) - - -class ReverseProxied(object): - '''Wrap the application in this middleware and configure the - front-end server to add these headers, to let you quietly bind - this to a URL other than / and to an HTTP scheme that is - different than what is used locally. - - location /myprefix { - proxy_pass http://192.168.0.1:5001; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /myprefix; - } - - :param app: the WSGI application - ''' - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - script_name = environ.get('HTTP_X_SCRIPT_NAME', '') - if script_name: - environ['SCRIPT_NAME'] = script_name - path_info = environ['PATH_INFO'] - if path_info.startswith(script_name): - environ['PATH_INFO'] = path_info[len(script_name):] - - scheme = environ.get('HTTP_X_SCHEME', '') - if scheme: - environ['wsgi.url_scheme'] = scheme - return self.app(environ, start_response) - - -# from https://github.com/noirbizarre/flask-restplus/issues/517 -def _register_apidoc(self, app): - conf = app.extensions.setdefault('restx', {}) - custom_apidoc = apidoc.Apidoc('restx_doc', 'flask_restx.apidoc', - template_folder='templates', - static_folder='static', - static_url_path="/swaggerui") - - @custom_apidoc.add_app_template_global - def swagger_static(filename): - return url_for('restx_doc.static', filename=filename) - - if not conf.get('apidoc_registered', False): - app.register_blueprint(custom_apidoc) - conf['apidoc_registered'] = True - - -# from https://github.com/noirbizarre/flask-restplus/pull/596/files -# make swagger work behind reverse proxy - -@cached_property -def __schema__(self): - ''' - The Swagger specifications/schema for this API - :returns dict: the schema as a serializable dict - ''' - if not self._schema: - try: - self._schema = Swagger(self).as_dict() - if self.behind_proxy and "host" in self._schema: - del self._schema["host"] - except Exception: - # Log the source exception for debugging purpose - # and return an error message - msg = 'Unable to render schema' - log.exception(msg) # This will provide a full traceback - return {'error': msg} - return self._schema - - -@property -def specs_url(self): - ''' - The Swagger specifications absolute url (ie. `swagger.json`) - Use a relative url when behind a proxy. - :rtype: str - ''' - if self.behind_proxy: - # Use relative URL. - external = False - else: - external = True - url = url_for(self.endpoint('specs'), _external=external) - # from https://github.com/noirbizarre/flask-restplus/pull/226/files - if self.app.config.get('SWAGGER_BASEPATH', ''): - prefix = url.split('/swagger.json')[0] - url = prefix + self.app.config.get('SWAGGER_BASEPATH', '') + '/swagger.json' - return url - return url - - -def inject_flasgger(app): - from flasgger import Swagger - with open("config/swagger.json") as infile: - specs = json.load(infile) - swagger = yaml.load(json.dumps(specs)) - swagger["host"] = os.getenv("HOST_IP", "localhost:5001") - if swagger["host"] == "localhost:5001": - swagger["schemes"] = ["http"] - Swagger(app, template=swagger, config=getSwaggerConfig()) - return app - - -def getSwaggerConfig(): - return { - "headers": [ - ], - "specs": [ - { - "endpoint": 'apispec', - "route": '/apispec.json', - "rule_filter": lambda rule: True, # all in - "model_filter": lambda tag: True, # all in - } - ], - "static_url_path": "/flasgger_static", - "static_folder": "static", # must be set by user - "swagger_ui": True, - "specs_route": "/" - } diff --git a/server/workers/base/run_base.py b/server/workers/base/run_base.py index f0c4b08f3..93f8146bc 100644 --- a/server/workers/base/run_base.py +++ b/server/workers/base/run_base.py @@ -10,8 +10,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "base_retrieval", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "base_retrieval" } redis_store = redis.StrictRedis(**redis_config) diff --git a/server/workers/base/src/base.py b/server/workers/base/src/base.py index 904b0fdad..bcd671f6a 100644 --- a/server/workers/base/src/base.py +++ b/server/workers/base/src/base.py @@ -1,3 +1,4 @@ +import os import json import subprocess import pandas as pd @@ -20,7 +21,7 @@ def next_item(self): endpoint = msg.get('endpoint') return k, params, endpoint - def execute_r(self, params): + def execute_search(self, params): q = params.get('q') service = params.get('service') data = {} @@ -52,6 +53,27 @@ def execute_r(self, params): self.logger.error(error) raise + def get_contentproviders(self): + runner = os.path.abspath(os.path.join(self.wd, "run_base_contentproviders.R")) + cmd = [self.command, runner, self.wd] + try: + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding='utf-8') + stdout, stderr = proc.communicate() + output = [o for o in stdout.split('\n') if len(o) > 0] + error = [o for o in stderr.split('\n') if len(o) > 0] + raw = json.loads(output[-1]) + if isinstance(raw, dict) and raw.get('status') == "error": + res = raw + else: + contentproviders = pd.DataFrame(raw) + res = {} + res["contentproviders"] = contentproviders.to_json(orient='records') + return res + except Exception as e: + self.logger.error(e) + self.logger.error(error) + def run(self): while True: k, params, endpoint = self.next_item() @@ -59,7 +81,7 @@ def run(self): self.logger.debug(params) if endpoint == "search": try: - res = self.execute_r(params) + res = self.execute_search(params) res["id"] = k if res.get("status") == "error" or params.get('raw') is True: self.redis_store.set(k+"_output", json.dumps(res)) @@ -68,3 +90,15 @@ def run(self): except Exception as e: self.logger.exception("Exception during data retrieval.") self.logger.error(params) + self.logger.error(e) + + if endpoint == "contentproviders": + try: + res = self.get_contentproviders() + res["id"] = k + self.redis_store.set(k+"_output", json.dumps(res)) + except Exception as e: + self.logger.exception("Exception during retrieval of contentproviders.") + self.logger.error(params) + self.logger.error(e) + diff --git a/server/workers/common/r_wrapper.py b/server/workers/common/r_wrapper.py index 5aaf8b887..cc3825e71 100644 --- a/server/workers/common/r_wrapper.py +++ b/server/workers/common/r_wrapper.py @@ -37,7 +37,7 @@ def add_default_params(self, params): def next_item(self): raise NotImplementedError - def execute_r(self, params): + def execute_search(self, params): raise NotImplementedError def run(self): @@ -49,7 +49,7 @@ def run(self): try: res = {} res["id"] = k - res["input_data"] = self.execute_r(params) + res["input_data"] = self.execute_search(params) res["params"] = params if params.get('raw') is True: redis_store.set(k+"_output", json.dumps(res)) diff --git a/server/workers/dataprocessing/run_dataprocessing.py b/server/workers/dataprocessing/run_dataprocessing.py index 375f8b603..e64a488d8 100644 --- a/server/workers/dataprocessing/run_dataprocessing.py +++ b/server/workers/dataprocessing/run_dataprocessing.py @@ -10,8 +10,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "dataprocessing", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "dataprocessing" } redis_store = redis.StrictRedis(**redis_config) diff --git a/server/workers/dataprocessing/src/headstart.py b/server/workers/dataprocessing/src/headstart.py index f2b15bda2..4923be7c3 100644 --- a/server/workers/dataprocessing/src/headstart.py +++ b/server/workers/dataprocessing/src/headstart.py @@ -52,7 +52,7 @@ def next_item(self): input_data = msg.get('input_data') return k, params, input_data - def execute_r(self, params, input_data): + def execute_search(self, params, input_data): q = params.get('q') service = params.get('service') data = {} @@ -92,7 +92,7 @@ def run(self): if params.get('vis_type') == "timeline": # the step of create_map can be dropped once deduplication is possible in API backend as well # TODO: create deduplicate endpoint in service worker and connect to that - metadata = self.execute_r(params, input_data) + metadata = self.execute_search(params, input_data) sg_data = sg.get_streamgraph_data(json.loads(metadata), params.get('q'), params.get('top_n', 12), @@ -103,7 +103,7 @@ def run(self): res["status"] = "success" self.redis_store.set(k+"_output", json.dumps(res)) else: - res = self.execute_r(params, input_data) + res = self.execute_search(params, input_data) self.redis_store.set(k+"_output", json.dumps(res)) except ValueError as e: self.logger.error(params) diff --git a/server/workers/flavorconfigs/example.env b/server/workers/flavorconfigs/example.env index ded6b8eec..d9c1e8133 100644 --- a/server/workers/flavorconfigs/example.env +++ b/server/workers/flavorconfigs/example.env @@ -13,7 +13,6 @@ REDIS_HOST=stable_redis_1 REDIS_PORT=6379 REDIS_DB=0 REDIS_PASSWORD=redis_password -REDIS_SSL=false LOGLEVEL=INFO BEHIND_PROXY=True diff --git a/server/workers/gsheets/run_gsheets.py b/server/workers/gsheets/run_gsheets.py index 4f507a4b7..63b1aef09 100644 --- a/server/workers/gsheets/run_gsheets.py +++ b/server/workers/gsheets/run_gsheets.py @@ -10,8 +10,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "gsheets_retrieval", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "gsheets_retrieval" } redis_store = redis.StrictRedis(**redis_config) gc = GSheetsClient(redis_store, os.environ.get("LOGLEVEL", "INFO")) diff --git a/server/workers/openaire/run_openaire.py b/server/workers/openaire/run_openaire.py index e752bf1b3..9bb54216e 100644 --- a/server/workers/openaire/run_openaire.py +++ b/server/workers/openaire/run_openaire.py @@ -10,8 +10,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "openaire_retrieval", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "openaire_retrieval" } redis_store = redis.StrictRedis(**redis_config) wrapper = OpenAIREClient("./other-scripts", "run_openaire.R", redis_store, diff --git a/server/workers/openaire/src/openaire.py b/server/workers/openaire/src/openaire.py index 7538d4d00..cc774eb7e 100644 --- a/server/workers/openaire/src/openaire.py +++ b/server/workers/openaire/src/openaire.py @@ -19,7 +19,7 @@ def next_item(self): endpoint = msg.get('endpoint') return k, params, endpoint - def execute_r(self, params): + def execute_search(self, params): q = params.get('acronym') service = params.get('service') data = {} @@ -52,7 +52,7 @@ def run(self): try: res = {} res["id"] = k - res["input_data"] = self.execute_r(params) + res["input_data"] = self.execute_search(params) res["params"] = params if params.get('raw') is True: self.redis_store.set(k+"_output", json.dumps(res)) diff --git a/server/workers/persistence/src/app.py b/server/workers/persistence/src/app.py index 9ba0d3fcd..6c155300d 100644 --- a/server/workers/persistence/src/app.py +++ b/server/workers/persistence/src/app.py @@ -4,9 +4,9 @@ from flask_restx import Api from flask_cors import CORS from werkzeug.middleware.proxy_fix import ProxyFix +import logging from apis.persistence import persistence_ns -import logging class ReverseProxied(object): '''Wrap the application in this middleware and configure the @@ -45,7 +45,7 @@ def api_patches(app): api_fixed = Api( app, title="Head Start API", - description="Head Start API demo", + description="Head Start API", version="0.1", prefix='/api', doc="/docs") @@ -63,6 +63,7 @@ def api_patches(app): api = api_patches(app) api.add_namespace(persistence_ns, path='/persistence') + app.logger.debug(app.config) app.logger.debug(app.url_map) diff --git a/server/workers/pubmed/run_pubmed.py b/server/workers/pubmed/run_pubmed.py index c4ab40ba5..4d1e734ef 100644 --- a/server/workers/pubmed/run_pubmed.py +++ b/server/workers/pubmed/run_pubmed.py @@ -10,8 +10,7 @@ "port": os.getenv("REDIS_PORT"), "db": os.getenv("REDIS_DB"), "password": os.getenv("REDIS_PASSWORD"), - "client_name": "pubmed_retrieval", - "ssl": True if os.getenv("REDIS_SSL") == "true" else False + "client_name": "pubmed_retrieval" } redis_store = redis.StrictRedis(**redis_config) wrapper = PubMedClient("./other-scripts", "run_pubmed.R", redis_store, diff --git a/server/workers/pubmed/src/pubmed.py b/server/workers/pubmed/src/pubmed.py index c8c792003..865f82ab8 100644 --- a/server/workers/pubmed/src/pubmed.py +++ b/server/workers/pubmed/src/pubmed.py @@ -19,7 +19,7 @@ def next_item(self): endpoint = msg.get('endpoint') return k, params, endpoint - def execute_r(self, params): + def execute_search(self, params): q = params.get('q') service = params.get('service') data = {} @@ -52,7 +52,7 @@ def run(self): try: res = {} res["id"] = k - res["input_data"] = self.execute_r(params) + res["input_data"] = self.execute_search(params) res["params"] = params if params.get('raw') is True: self.redis_store.set(k+"_output", json.dumps(res)) diff --git a/vis/js/components/ContextLine.js b/vis/js/components/ContextLine.js index 9ca8f4edd..ecc214501 100644 --- a/vis/js/components/ContextLine.js +++ b/vis/js/components/ContextLine.js @@ -52,8 +52,10 @@ class ContextLine extends React.Component { {defined(params.dataSource) && ( )} {defined(params.timespan) && {params.timespan}} diff --git a/vis/js/reducers/contextLine.js b/vis/js/reducers/contextLine.js index 8a1a5de13..262e3e349 100644 --- a/vis/js/reducers/contextLine.js +++ b/vis/js/reducers/contextLine.js @@ -38,6 +38,7 @@ const contextLine = (state = {}, action) => { typeof config.service_name !== "undefined" ? config.service_name : config.service_names[context.service], + contentProvider: context.params ? context.params.repo_name : null, paperCount: config.create_title_from_context_style === "viper" ? papers.filter((p) => p.resulttype.includes("publication")).length diff --git a/vis/js/reducers/modals.js b/vis/js/reducers/modals.js index cd7517cba..b4a956b2c 100644 --- a/vis/js/reducers/modals.js +++ b/vis/js/reducers/modals.js @@ -44,8 +44,8 @@ const modals = ( openCitationModal: false, citedPaper: null, exportedPaper: null, - showTwitterButton: !!action.configObject.show_twitter_button, - showEmailButton: !!action.configObject.show_email_button, + showTwitterButton: showTwitterShare(action.configObject, action.contextObject), + showEmailButton: showEmailShare(action.configObject, action.contextObject), }; case "OPEN_EMBED_MODAL": return { @@ -148,3 +148,17 @@ const getSheetID = (config, context) => { return config.files[0].file; }; + +const showTwitterShare = (config, context) => { + if (config.credit_embed) { + return false; + } + return !!config.show_twitter_button; +}; + +const showEmailShare = (config, context) => { + if (config.credit_embed) { + return false; + } + return !!config.show_email_button; +}; diff --git a/vis/js/templates/contextfeatures/DataSource.jsx b/vis/js/templates/contextfeatures/DataSource.jsx index b4a7d362b..9558c52c3 100644 --- a/vis/js/templates/contextfeatures/DataSource.jsx +++ b/vis/js/templates/contextfeatures/DataSource.jsx @@ -1,13 +1,36 @@ import React from "react"; +import { shorten } from "../../utils/string"; +import HoverPopover from "../HoverPopover"; + +const MAX_CONTENT_PROVIDER_LENGTH = 6; + +const DataSource = ({ label, source, contentProvider, popoverContainer }) => { + if (contentProvider) { + const content = shorten(contentProvider, MAX_CONTENT_PROVIDER_LENGTH); + + return ( + // html template starts here + + {label}:{" "} + + {content} + + + // html template ends here + ); + } -const DataSource = ({ label, value }) => { return ( // html template starts here // html template ends here diff --git a/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx b/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx index 25eb6f62d..1f0262f25 100644 --- a/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx +++ b/vis/js/templates/modals/infomodal/subcomponents/DataSource.jsx @@ -1,13 +1,24 @@ import React from "react"; -const DataSource = ({ source, description, logo }) => { +const DataSource = ({ source, contentProvider, description, logo }) => { + const getDataSource = () => { + if (!contentProvider) { + return {source}; + } + + return ( + <> + {contentProvider} (via {source}) + + ); + }; + return ( // html template starts here <>

Data source

- The data is taken from {source}.{" "} - {description} + The data is taken from {getDataSource()}. {description}

{!!logo &&

{logo}

} diff --git a/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx b/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx index a9978cad5..39d4284e9 100644 --- a/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx +++ b/vis/js/templates/modals/infomodal/subcomponents/StandardKMInfo.jsx @@ -8,7 +8,7 @@ const StandardKMInfo = ({ serviceName, serviceDesc, serviceLogo, - params: { query, customTitle }, + params: { query, customTitle, repo_name }, }) => { return ( // html template starts here @@ -73,6 +73,7 @@ const StandardKMInfo = ({ {!!serviceName && ( diff --git a/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx b/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx index edb3973ab..c6daf26da 100644 --- a/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx +++ b/vis/js/templates/modals/infomodal/subcomponents/StandardSGInfo.jsx @@ -8,7 +8,7 @@ const StandardSGInfo = ({ serviceName, serviceDesc, serviceLogo, - params: { query, customTitle }, + params: { query, customTitle, repo_name }, }) => { return ( // html template starts here @@ -50,6 +50,7 @@ const StandardSGInfo = ({ {!!serviceName && ( diff --git a/vis/js/utils/string.js b/vis/js/utils/string.js index 893a8f178..c0deaeaaf 100644 --- a/vis/js/utils/string.js +++ b/vis/js/utils/string.js @@ -71,6 +71,22 @@ export const stringCompare = (a, b, sort_order) => { return d3.ascending(a, b); }; +/** + * Shortens the input string to selected length. + * @param {String} string input to shorten + * @param {Number} length output length + * @param {String} end the ending if the string was shortened + * + * @returns shortened string + */ + export const shorten = (string, length, end = "...") => { + if (string.length <= length) { + return string; + } + + return string.slice(0, length) + end; +}; + export const formatString = (string, params) => { Object.keys(params).forEach((param) => { string = string.replaceAll("${" + param + "}", params[param]);