Skip to content

Commit

Permalink
add solrcloud proxy to support authorized requests for ckan 2.6, 2.7
Browse files Browse the repository at this point in the history
  • Loading branch information
OriHoch committed Jan 24, 2019
1 parent ce53e36 commit afde17b
Show file tree
Hide file tree
Showing 6 changed files with 162 additions and 3 deletions.
51 changes: 51 additions & 0 deletions ckan_cloud_operator/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import os
import click
import time
import datetime
from urllib.parse import urlparse
from xml.etree import ElementTree
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
from ckan_cloud_operator.infra import CkanInfra
Expand Down Expand Up @@ -299,6 +301,55 @@ def ckan_infra_cloudsql_proxy():
subprocess.check_call(f'kubectl -n ckan-cloud port-forward deployment/cloudsql-proxy 5432', shell=True)


@ckan_infra.command('deploy-solr-proxy')
def deploy_ckan_infra_solr_proxy():
"""Deploys a proxy inside the cluster which allows to access the centralized solr without authentication"""
labels = {'app': 'ckan-cloud-solrcloud-proxy'}
infra = CkanInfra()
solr_url = urlparse(infra.SOLR_HTTP_ENDPOINT)
scheme = solr_url.scheme
hostname = solr_url.hostname
port = solr_url.port
if not port:
port = '443' if scheme == 'https' else '8983'
kubectl.update_secret('solrcloud-proxy', {
'SOLR_URL': f'{scheme}://{hostname}:{port}',
'SOLR_USER': infra.SOLR_USER,
'SOLR_PASSWORD': infra.SOLR_PASSWORD
})
kubectl.apply(kubectl.get_deployment('solrcloud-proxy', labels, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {'type': 'RollingUpdate',},
'template': {
'metadata': {
'labels': labels,
'annotations': {
'ckan-cloud/operator-timestamp': str(datetime.datetime.now())
}
},
'spec': {
'containers': [
{
'name': 'solrcloud-proxy',
'image': 'viderum/ckan-cloud-operator-solrcloud-proxy',
'envFrom': [{'secretRef': {'name': 'solrcloud-proxy'}}],
'ports': [{'containerPort': 8983}],
}
]
}
}
}))
service = kubectl.get_resource('v1', 'Service','solrcloud-proxy', labels)
service['spec'] = {
'ports': [
{'name': '8983', 'port': 8983}
],
'selector': labels
}
kubectl.apply(service)


#################################
#### ####
#### deis-instance ####
Expand Down
9 changes: 6 additions & 3 deletions ckan_cloud_operator/deis_ckan/envvars.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ def _update(self):
datastore_password = self.instance.annotations.get_secret('datastorePassword')
datastore_ro_user = self.instance.annotations.get_secret('datastoreReadonlyUser')
datastore_ro_password = self.instance.annotations.get_secret('datatastoreReadonlyPassword')
solr_http_endpoint = ckan_infra.SOLR_HTTP_ENDPOINT_SIMPLE
# solr_http_endpoint = ckan_infra.SOLR_HTTP_ENDPOINT_SIMPLE
# a proxy solr to support authenticated requests for ckan 2.6, 2.7
solr_http_endpoint = 'http://solrcloud-proxy.ckan-cloud:8983/solr'
solr_collection_name = spec.solrCloudCollection['name']
if 'fromSecret' in spec.envvars:
envvars = kubectl.get(f'secret {spec.envvars["fromSecret"]}')
Expand All @@ -47,8 +49,9 @@ def _update(self):
CKAN__DATASTORE__READ_URL=f"postgresql://{datastore_ro_user}:{datastore_ro_password}@{postgres_host}:5432/{datastore_name}",
CKAN__DATASTORE__WRITE_URL=f"postgresql://{datastore_name}:{datastore_password}@{postgres_host}:5432/{datastore_name}",
CKAN_SOLR_URL=f"{solr_http_endpoint}/{solr_collection_name}",
CKAN_SOLR_USER=ckan_infra.SOLR_USER,
CKAN_SOLR_PASSWORD=ckan_infra.SOLR_PASSWORD,
# we are using the non-authenticated proxy, so this has to be disabled to prevent ckans which support auth from using them
# CKAN_SOLR_USER=ckan_infra.SOLR_USER,
# CKAN_SOLR_PASSWORD=ckan_infra.SOLR_PASSWORD,
CKANEXT__S3FILESTORE__AWS_STORAGE_PATH=storage_path,
CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=ckan_infra.GCLOUD_STORAGE_ACCESS_KEY_ID,
CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=ckan_infra.GCLOUD_STORAGE_SECRET_ACCESS_KEY,
Expand Down
3 changes: 3 additions & 0 deletions solrcloud-proxy/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM nginx
COPY default.conf.template entrypoint.sh templater.sh /
CMD ["/entrypoint.sh"]
11 changes: 11 additions & 0 deletions solrcloud-proxy/default.conf.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
server {
listen 8983;
server_name _;
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass {{SOLR_URL}};
proxy_set_header Authorization "Basic {{SOLR_AUTH}}";
}
}
8 changes: 8 additions & 0 deletions solrcloud-proxy/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

export SOLR_URL
export SOLR_AUTH="$(echo ${SOLR_USER}:${SOLR_PASSWORD} | base64)"
bash /templater.sh /default.conf.template > /etc/nginx/conf.d/default.conf
echo Proxying requests from port 8983 to ${SOLR_URL} ${SOLR_USER} ${SOLR_PASSWORD}
cd /etc/nginx
exec nginx -g "daemon off;"
83 changes: 83 additions & 0 deletions solrcloud-proxy/templater.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# https://github.com/johanhaleby/bash-templater/commit/5ac655d554238ac70b08ee4361d699ea9954c941
readonly PROGNAME=$(basename $0)
config_file="<none>"
print_only="false"
silent="false"
[ $# -eq 0 ] && exit 1
[[ ! -f "${1}" ]] && exit 1
template="${1}"
if [ "$#" -ne 0 ]; then
while [ "$#" -gt 0 ]
do
case "$1" in
-p|--print)
print_only="true"
;;
-f|--file)
config_file="$2"
;;
-s|--silent)
silent="true"
;;
--)
break
;;
-*)
exit 1
;;
*) ;;
esac
shift
done
fi
vars=$(grep -oE '\{\{[A-Za-z0-9_]+\}\}' "${template}" | sort | uniq | sed -e 's/^{{//' -e 's/}}$//')
if [[ -z "$vars" ]]; then
if [ "$silent" == "false" ]; then
echo "Warning: No variable was found in ${template}, syntax is {{VAR}}" >&2
fi
fi
if [ "${config_file}" != "<none>" ]; then
if [[ ! -f "${config_file}" ]]; then
echo "The file ${config_file} does not exists" >&2
echo "$usage"
exit 1
fi
tmpfile=`mktemp`
sed -e "s;\&;\\\&;g" -e "s;\ ;\\\ ;g" "${config_file}" > $tmpfile
source $tmpfile
fi
var_value() {
eval echo \$$1
}
replaces=""
defaults=$(grep -oE '^\{\{[A-Za-z0-9_]+=.+\}\}' "${template}" | sed -e 's/^{{//' -e 's/}}$//')
for default in $defaults; do
var=$(echo "$default" | grep -oE "^[A-Za-z0-9_]+")
current=`var_value $var`
if [[ -z "$current" ]]; then
eval $default
fi
replaces="-e '/^{{$var=/d' $replaces"
vars="$vars
$current"
done
vars=$(echo $vars | sort | uniq)
if [[ "$print_only" == "true" ]]; then
for var in $vars; do
value=`var_value $var`
echo "$var = $value"
done
exit 0
fi
for var in $vars; do
value=`var_value $var`
if [[ -z "$value" ]]; then
if [ $silent == "false" ]; then
echo "Warning: $var is not defined and no default is set, replacing by empty" >&2
fi
fi
value=$(echo "$value" | sed 's/\//\\\//g');
replaces="-e 's/{{$var}}/${value}/g' $replaces"
done
escaped_template_path=$(echo $template | sed 's/ /\\ /g')
eval sed $replaces "$escaped_template_path"

0 comments on commit afde17b

Please sign in to comment.