Transfer command

.flake8

@@ -1,5 +1,5 @@
 [flake8]
 ignore = E203, E266, E501, W503
-max-line-length = 80
+max-line-length = 119
 max-complexity = 18
 select = B,C,E,F,W,T4,B9

.travis.yml

@@ -2,14 +2,20 @@ dist: xenial
 language: python
 cache: pip
 env:
-- TEST_CMD="pytest tests/ --integration --cov=esque --local"
+- >
+  TEST_CMD="pytest tests/ --integration --cov=esque --local"
+  BROKER_URL="localhost:9092"
+  ZOOKEEPER_URL="localhost:2181"
+  SCHEMA_REGISTRY_URL="localhost:8081"
 before_install:
-- wget https://mirror.netcologne.de/apache.org/kafka/2.2.0/kafka_2.12-2.2.0.tgz -O kafka.tgz
-- mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1
-- nohup bash -c "cd kafka && bin/zookeeper-server-start.sh config/zookeeper.properties  &"
-- nohup bash -c "cd kafka && bin/kafka-server-start.sh config/server.properties &"
-- scripts/wait-for-it.sh localhost:9092
-- scripts/wait-for-it.sh localhost:2181
+- wget http://packages.confluent.io/archive/5.2/confluent-community-5.2.1-2.12.tar.gz -O confluent-community.tgz
+- mkdir -p confluent-community && tar xzf confluent-community.tgz -C confluent-community --strip-components 1
+- confluent-community/bin/zookeeper-server-start -daemon confluent-community/etc/kafka/zookeeper.properties
+- bash -c "scripts/wait-for-it.sh ${ZOOKEEPER_URL} -t 60"
+- confluent-community/bin/kafka-server-start -daemon confluent-community/etc/kafka/server.properties
+- bash -c "scripts/wait-for-it.sh ${BROKER_URL} -t 60"
+- confluent-community/bin/schema-registry-start -daemon confluent-community/etc/schema-registry/schema-registry.properties
+- bash -c "scripts/wait-for-it.sh ${SCHEMA_REGISTRY_URL} -t 60"
 install:
 - pip install coverage coveralls flake8 pipenv
 - pipenv install --system --dev --deploy
@@ -26,11 +32,13 @@ notifications:
 matrix:
   include:
   - name: black
+    before_install: []
     python: 3.7
     env:
-    - TEST_CMD="black --check --verbose ."
+    - TEST_CMD="black --check --verbose --line-length=119 ."
   - name: flake8
     python: 3.7
+    before_install: []
     env:
     - TEST_CMD="flake8 esque/"
   - name: '3.6'

README.md

@@ -134,6 +134,17 @@ While this `docker-compose` stack is up, you can run the tests from the CLI via
 Alternatively, you can also run the entire test suite, without needing to setup the development environment, in docker compose via `docker-compose -f docker-compose.yml -f docker-compose.test.yml` 
 
 
+### Pre Commit Hooks
+
+To install pre commit hooks run:
+
+```
+pip install pre-commit
+pre-commit install
+pre-commit install-hooks`
+``
+
+
 ## Alternatives
 
 - [LinkedIn KafkaTools](https://github.com/linkedin/kafka-tools)

config/sample_config.cfg

@@ -5,8 +5,10 @@ current = docker
 bootstrap_hosts = localhost
 bootstrap_port = 9092
 security_protocol = LOCAL
+schema_registry = http://localhost:8081
 
 [Context.docker]
 bootstrap_hosts = kafka
 bootstrap_port = 9093
 security_protocol = PLAINTEXT
+schema_registry = http://schema_registry:8081

docker-compose.test.yml

@@ -8,7 +8,8 @@ services:
       ESQUE_TEST_ENV: "ci"
     depends_on:
       - kafka
+      - schema_registry
     command: >
       -c
         "(until (kafkacat -b kafka:9093 -X debug=all -L); do sleep 5s; done) \
-        && pytest tests/ --integration"
+        && python3 -u -m pytest -v -x tests/ --integration"

docker-compose.yml

@@ -37,5 +37,15 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
 
-
-
+  schema_registry:
+    image: confluentinc/cp-schema-registry:5.2.2
+    container_name: schema_registry
+    environment:
+      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
+      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
+      - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
+      - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=PLAINTEXT://kafka:9093
+    depends_on:
+      - kafka
+    ports:
+      - 8081:8081