-
Notifications
You must be signed in to change notification settings - Fork 206
/
fully-managed-sftp-source-json.sh
executable file
·109 lines (84 loc) · 3.28 KB
/
fully-managed-sftp-source-json.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
#!/bin/bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source ${DIR}/../../scripts/utils.sh
NGROK_AUTH_TOKEN=${NGROK_AUTH_TOKEN:-$1}
display_ngrok_warning
bootstrap_ccloud_environment
set +e
playground topic delete --topic sftp-testing-topic
set -e
docker compose build
docker compose down -v --remove-orphans
docker compose up -d --quiet-pull
sleep 5
docker exec sftp-server bash -c "
mkdir -p /chroot/home/foo/upload/input
mkdir -p /chroot/home/foo/upload/error
mkdir -p /chroot/home/foo/upload/finished
chown -R foo /chroot/home/foo/upload
"
echo $'{"id":1,"first_name":"Roscoe","last_name":"Brentnall","email":"rbrentnall0@mediafire.com","gender":"Male","ip_address":"202.84.142.254","last_login":"2018-02-12T06:26:23Z","account_balance":1450.68,"country":"CZ","favorite_color":"#4eaefa"}\n{"id":2,"first_name":"Gregoire","last_name":"Fentem","email":"gfentem1@nsw.gov.au","gender":"Male","ip_address":"221.159.106.63","last_login":"2015-03-27T00:29:56Z","account_balance":1392.37,"country":"ID","favorite_color":"#e8f686"}' > json-sftp-source.json
docker cp json-sftp-source.json sftp-server:/chroot/home/foo/upload/input/
rm -f json-sftp-source.json
log "Waiting for ngrok to start"
while true
do
container_id=$(docker ps -q -f name=ngrok)
if [ -n "$container_id" ]
then
status=$(docker inspect --format '{{.State.Status}}' $container_id)
if [ "$status" = "running" ]
then
log "Getting ngrok hostname and port"
NGROK_URL=$(curl --silent http://127.0.0.1:4040/api/tunnels | jq -r '.tunnels[0].public_url')
NGROK_HOSTNAME=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 1)
NGROK_PORT=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 2)
if ! [[ $NGROK_PORT =~ ^[0-9]+$ ]]
then
log "NGROK_PORT is not a valid number, keep retrying..."
continue
else
break
fi
fi
fi
log "Waiting for container ngrok to start..."
sleep 5
done
connector_name="SftpSource_$USER"
set +e
playground connector delete --connector $connector_name > /dev/null 2>&1
set -e
log "Creating fully managed connector"
playground connector create-or-update --connector $connector_name << EOF
{
"connector.class": "SftpSource",
"name": "$connector_name",
"kafka.auth.mode": "KAFKA_API_KEY",
"kafka.api.key": "$CLOUD_KEY",
"kafka.api.secret": "$CLOUD_SECRET",
"output.data.format": "JSON",
"input.file.parser.format": "JSON",
"kafka.topic": "sftp-testing-topic",
"schema.generation.enabled": "true",
"schema.generation.key.name": "key",
"schema.generation.value.name": "value",
"input.path": "/home/foo/upload/input",
"error.path": "/home/foo/upload/error",
"finished.path": "/home/foo/upload/finished",
"input.file.pattern": ".*\\\\.json",
"sftp.username":"foo",
"sftp.password":"pass",
"sftp.host":"$NGROK_HOSTNAME",
"sftp.port":"$NGROK_PORT",
"tasks.max" : "1"
}
EOF
wait_for_ccloud_connector_up $connector_name 180
sleep 5
log "Verify we have received the data in sftp-testing-topic topic"
playground topic consume --topic sftp-testing-topic --min-expected-messages 2 --timeout 60
log "Do you want to delete the fully managed connector $connector_name ?"
check_if_continue
playground connector delete --connector $connector_name