Skip to content
Permalink
Browse files
[BAHIR-92] Add Flink extensions in update-doc.sh
Refactoring update-doc.sh script to include Bahir-Flink extensions
and make it easier to add future extensions.

Closes #8
  • Loading branch information
tzulitai authored and ckadner committed Mar 6, 2017
1 parent 0d53ccd commit 4142e1e34462c3037e0db5366bee28f27b00ff37
Showing 6 changed files with 212 additions and 40 deletions.
@@ -0,0 +1,27 @@
---
layout: page
title: Apache Flink Streaming Connector for ActiveMQ
description: Apache Flink Streaming Connector for ActiveMQ
group: nav-right
---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->

{% include JB/setup %}

@@ -0,0 +1,27 @@
---
layout: page
title: Apache Flink Streaming Connector for Akka
description: Apache Flink Streaming Connector for Akka
group: nav-right
---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->

{% include JB/setup %}

@@ -0,0 +1,27 @@
---
layout: page
title: Apache Flink Streaming Connector for Apache Flume
description: Apache Flink Streaming Connector for Apache Flume
group: nav-right
---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->

{% include JB/setup %}

@@ -0,0 +1,27 @@
---
layout: page
title: Apache Flink Streaming Connector for Netty
description: Apache Flink Streaming Connector for Netty
group: nav-right
---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->

{% include JB/setup %}

@@ -0,0 +1,27 @@
---
layout: page
title: Apache Flink Streaming Connector for Redis
description: Apache Flink Streaming Connector for Redis
group: nav-right
---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->

{% include JB/setup %}

@@ -43,62 +43,99 @@
set -e

BASE_DIR=$(pwd)
WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates
WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current
BAHIR_SOURCE_DIR=$BASE_DIR/target/bahir

SPARK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates
SPARK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current
SPARK_REPO_NAME=bahir
SPARK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$SPARK_REPO_NAME

FLINK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/flink/templates
FLINK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/flink/current
FLINK_REPO_NAME=bahir-flink
FLINK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$FLINK_REPO_NAME

function checkout_code {
# Checkout code
cd "$BASE_DIR" # make sure we're in the base dir

rm -rf target
mkdir target
cd target
git clone https://git-wip-us.apache.org/repos/asf/bahir.git --quiet
cd bahir
git checkout $GIT_REF
git_hash=`git rev-parse --short HEAD`
echo "Checked out Bahir git hash $git_hash"
git clone "https://git-wip-us.apache.org/repos/asf/$1.git" --quiet
cd "$1"
git checkout "$GIT_REF"
git_hash=$(git rev-parse --short HEAD)
echo "Checked out $1 git hash $git_hash"

git clean -d -f -x

cd "$BASE_DIR" #return to base dir
cd "$BASE_DIR" # return to base dir
}

checkout_code
function update_docs {
WEBSITE_TEMPLATES_DIR=$1
WEBSITE_DOC_DIR=$2
SOURCE_DIR=$3
shift 3

[ ! -d "$WEBSITE_DOC_DIR" ] && mkdir "$WEBSITE_DOC_DIR"

while [ $# -ne 0 ]; do
echo "Syncing document $WEBSITE_DOC_DIR/$1.md with source $SOURCE_DIR/$2/README.md"
rm -f "$WEBSITE_DOC_DIR/$1.md"
cp "$WEBSITE_TEMPLATES_DIR/$1.template" "$WEBSITE_DOC_DIR/$1.md"
cat "$SOURCE_DIR/$2/README.md" >> "$WEBSITE_DOC_DIR/$1.md"
shift 2
done
}

rm -rf $WEBSITE_DOC_DIR/spark*.md
function check_version_strings {
if grep -q -r "[0-9]-SNAPSHOT" "$1"/*.md ; then
echo
echo "TODO: Replace '...-SNAPSHOT' version strings:"
echo
grep -r -n "[0-9]-SNAPSHOT" "$1"/*.md | sed -e 's|'"$(pwd)"/'||g' | grep --color "[0-9.]*-SNAPSHOT"
echo
echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
echo
echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $1/*.md"
echo
else
echo
echo "Generated files:"
echo
ls "$1"/*.md | xargs -n1 | sed -e 's|'"$(pwd -P)"/'||g'
fi
}

cp $WEBSITE_TEMPLATES_DIR/spark-sql-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
cat $BAHIR_SOURCE_DIR/sql-streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
echo
echo "================= Updating Apache Spark Extension documents ================="
echo

cp $WEBSITE_TEMPLATES_DIR/spark-streaming-akka.template $WEBSITE_DOC_DIR/spark-streaming-akka.md
cat $BAHIR_SOURCE_DIR/streaming-akka/README.md >> $WEBSITE_DOC_DIR/spark-streaming-akka.md
checkout_code $SPARK_REPO_NAME

cp $WEBSITE_TEMPLATES_DIR/spark-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
cat $BAHIR_SOURCE_DIR/streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
update_docs "$SPARK_WEBSITE_TEMPLATES_DIR" "$SPARK_WEBSITE_DOC_DIR" "$SPARK_BAHIR_SOURCE_DIR" \
spark-sql-streaming-mqtt sql-streaming-mqtt \
spark-streaming-akka streaming-akka \
spark-streaming-mqtt streaming-mqtt \
spark-streaming-twitter streaming-twitter \
spark-streaming-zeromq streaming-zeromq

cp $WEBSITE_TEMPLATES_DIR/spark-streaming-twitter.template $WEBSITE_DOC_DIR/spark-streaming-twitter.md
cat $BAHIR_SOURCE_DIR/streaming-twitter/README.md >> $WEBSITE_DOC_DIR/spark-streaming-twitter.md
check_version_strings "$SPARK_WEBSITE_DOC_DIR"

cp $WEBSITE_TEMPLATES_DIR/spark-streaming-zeromq.template $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
cat $BAHIR_SOURCE_DIR/streaming-zeromq/README.md >> $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
echo
echo "================= Updating Apache Flink Extension documents ================="
echo

set +e
checkout_code $FLINK_REPO_NAME

update_docs "$FLINK_WEBSITE_TEMPLATES_DIR" "$FLINK_WEBSITE_DOC_DIR" "$FLINK_BAHIR_SOURCE_DIR" \
flink-streaming-activemq flink-connector-activemq \
flink-streaming-akka flink-connector-akka \
flink-streaming-flume flink-connector-flume \
flink-streaming-netty flink-connector-netty \
flink-streaming-redis flink-connector-redis

echo "Done."

if grep -q -r "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md ; then
echo
echo "TODO: Replace '...-SNAPSHOT' version strings:"
echo
grep -r -n "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md | sed -e 's|'$(pwd)/'||g' | grep --color "[0-9.]*-SNAPSHOT"
echo
echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
echo
echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $WEBSITE_DOC_DIR/spark*.md"
echo
else
echo
echo "Generated files:"
echo
ls $WEBSITE_DOC_DIR/spark*.md | xargs -n1 | sed -e 's|'$(pwd -P)/'||g'
fi
check_version_strings "$FLINK_WEBSITE_DOC_DIR"

set +e

0 comments on commit 4142e1e

Please sign in to comment.