Skip to content
Permalink
Browse files

remove rabbitmq, queue in redis lists, add keys to message instead of…

… limit/offset quries
  • Loading branch information...
tsurowiec committed Oct 19, 2018
1 parent b6d00b3 commit f8f7e062c08d4d41aa768db362f8666cbc6d20db
Showing with 570 additions and 677 deletions.
  1. +0 −5 .env.dist
  2. +1 −1 Makefile
  3. +3 −10 README.md
  4. +2 −2 behat.yml.dist
  5. +0 −1 composer.json
  6. +138 −278 composer.lock
  7. +0 −1 config/bundles.php
  8. +2 −0 config/packages/doctrine.yaml
  9. +0 −15 config/packages/old_sound_rabbit_mq.yaml
  10. +0 −15 config/packages/test/old_sound_rabbit_mq.yaml
  11. +1 −5 config/services.yaml
  12. +0 −8 config/services_test.yaml
  13. +3 −10 docker-compose.yml
  14. +37 −0 features/Run/Data/copyUTF8.feature
  15. +15 −7 features/bootstrap/{ChunkCounterContext.php → ChunkCacheContext.php}
  16. +7 −17 features/bootstrap/CommandContext.php
  17. +66 −0 src/Command/ConsumerCommand.php
  18. +19 −10 src/Command/FinishCommand.php
  19. +15 −14 src/Command/RunCommand.php
  20. +75 −0 src/Fogger/Data/ChunkCache.php
  21. +7 −17 src/Fogger/Data/ChunkConsumer.php
  22. +0 −44 src/Fogger/Data/ChunkCounter.php
  23. +0 −55 src/Fogger/Data/ChunkDivider.php
  24. +7 −25 src/Fogger/Data/ChunkMessage.php
  25. +29 −25 src/Fogger/Data/ChunkProducer.php
  26. +7 −14 src/Fogger/Data/ChunkReader.php
  27. +3 −2 src/Fogger/Data/DataCopier.php
  28. +61 −0 src/Fogger/Data/SourceQuery.php
  29. +0 −32 src/Fogger/Data/TableQuery.php
  30. +7 −2 src/Fogger/Mask/AbstractCachedMask.php
  31. +4 −4 src/Fogger/Mask/FakerMask.php
  32. +5 −5 src/Fogger/Recipe/RecipeFactory.php
  33. +6 −3 src/Fogger/Recipe/RecipeTableFactory.php
  34. +12 −9 src/Fogger/Refine/Refiner.php
  35. +13 −0 src/Fogger/Schema/SchemaManipulator.php
  36. +11 −19 src/Fogger/Subset/AbstratctHeadOrTailSubset.php
  37. +4 −3 src/Fogger/Subset/HeadSubset.php
  38. +2 −1 src/Fogger/Subset/NoSubset.php
  39. +3 −2 src/Fogger/Subset/RangeSubset.php
  40. +1 −1 src/Fogger/Subset/SubsetStrategyInterface.php
  41. +4 −3 src/Fogger/Subset/TailSubset.php
  42. +0 −12 symfony.lock
@@ -8,11 +8,6 @@ APP_SECRET=1b90fda83888e1852b735fdf9d37cf40
SOURCE_DATABASE_URL=mysql://user:pass@source/source
TARGET_DATABASE_URL=mysql://user:pass@target/target
###< doctrine/doctrine-bundle ###

###> php-amqplib/rabbitmq-bundle ###
RABBITMQ_URL=amqp://user:pass@rabbit:5672
###< php-amqplib/rabbitmq-bundle ###

###> snc/redis-bundle ###
# passwords that contain special characters (@, %, :, +) must be urlencoded
REDIS_URL=redis://redis
@@ -15,7 +15,7 @@ install:
start:
${DOCKER_COMPOSE} up -d
echo "waiting for services to start..."
sleep 30
sleep 16

.PHONY: stop
stop:
@@ -10,7 +10,7 @@ You can configure various masking and subsetting strategies, and when what *fogg

## How to use the docker image

*Fogger* requires docker environment, redis and rabbitMq services and two databases: source and target. You can set up this stack using for example this docker-compose file:
*Fogger* requires docker environment, redis for caching and two databases: source and target. You can set up this stack using for example this docker-compose file:
```
version: '2.0'
services:
@@ -21,24 +21,17 @@ services:
environment:
SOURCE_DATABASE_URL: mysql://user:pass@source:3306/source
TARGET_DATABASE_URL: mysql://user:pass@target:3306/target
RABBITMQ_URL: amqp://user:pass@rabbit:5672
REDIS_URL: redis://redis
worker:
image: tshio/fogger:latest
environment:
SOURCE_DATABASE_URL: mysql://user:pass@source:3306/source
TARGET_DATABASE_URL: mysql://user:pass@target:3306/target
RABBITMQ_URL: amqp://user:pass@rabbit:5672
REDIS_URL: redis://redis
restart: always
command: rabbit:consumer --messages=200 fogger_data_chunks
command: fogger:consumer --messages=200
redis:
image: redis:4
rabbit:
image: rabbitmq:3
environment:
RABBITMQ_DEFAULT_USER: user
RABBITMQ_DEFAULT_PASS: pass
source:
volumes:
- ./dump.sql:/docker-entrypoint-initdb.d/dump.sql
@@ -109,7 +102,7 @@ For the clarity and readability of the config files, all the tables that will no
* faker - will use a marvelous [faker](https://github.com/fzaninotto/Faker) library. Pass the `method` of faker that you want to use here as an option.

`email: { maskStrategy: "faker", options: { method: "safeEmail" }`
`date: { maskStrategy: "faker", options: { method: "date", parameters: ["Y::m::d", "2017-12-31 23:59:59"] }`
`date: { maskStrategy: "faker", options: { method: "date", arguments: ["Y::m::d", "2017-12-31 23:59:59"] }`

#### Subsetting data

@@ -10,8 +10,8 @@ default:
- DatabaseContext:
source: '@doctrine.dbal.source_connection'
target: '@doctrine.dbal.target_connection'
- ChunkCounterContext:
chunkCounter: '@App\Fogger\Data\ChunkCounter'
- ChunkCacheContext:
chunkCache: '@App\Fogger\Data\ChunkCache'

extensions:
Behat\Symfony2Extension:
@@ -7,7 +7,6 @@
"ext-iconv": "*",
"ext-json": "*",
"fzaninotto/faker": "^1.8",
"php-amqplib/rabbitmq-bundle": "^1.14",
"predis/predis": "^1.1",
"snc/redis-bundle": "^2.1",
"symfony/console": "*",
Oops, something went wrong.

0 comments on commit f8f7e06

Please sign in to comment.
You can’t perform that action at this time.