Skip to content
This repository has been archived by the owner on Jun 25, 2020. It is now read-only.

Commit

Permalink
reduce solr downtime with commit=false #49
Browse files Browse the repository at this point in the history
  • Loading branch information
@felixlohmeier
felixlohmeier committed Nov 28, 2018
1 parent d9f92a9 commit f3c96f0
Show file tree
Hide file tree
Showing 10 changed files with 10 additions and 10 deletions.
2 changes: 1 addition & 1 deletion bin/copernicus.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/ediss.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/haw.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/hcu.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/hsu.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/tubdok.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/tuhh-fdm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/uhh-fis.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion bin/uke-fis.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary "{ \"delete\": { \"query\": \"collectionId:${codename}\" } }" | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=${codename}&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/02_transformed/${codename}_${date}.tsv | jq .responseHeader
Expand Down
2 changes: 1 addition & 1 deletion run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ if [ -n "$solr_url" ]; then
multivalue_config=$(printf %s "${multivalue_config[@]}")
for i in ${solr_url[@]}; do
echo "delete existing data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update?commit=true" -H "Content-Type: application/json" --data-binary '{ "delete": { "query": "*:*" } }' | jq .responseHeader
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) -sS "${i}/update" -H "Content-Type: application/json" --data-binary '{ "delete": { "query": "*:*" } }' | jq .responseHeader
echo ""
echo "load new data in ${i}"
curl $(if [ -n "$solr_user" ]; then echo "-u ${solr_user}:${solr_pass}"; fi) --progress-bar "${i}/update/csv?commit=true&optimize=true&separator=%09&literal.collectionId=hos&split=true${multivalue_config}" --data-binary @- -H 'Content-type:text/plain; charset=utf-8' < ${data_dir}/03_combined/all_${date}.tsv | jq .responseHeader
Expand Down

0 comments on commit f3c96f0

Please sign in to comment.