Data Exporter

.env

@@ -29,6 +29,8 @@ REDIS_HOST="localhost"
 
 APPLICATION_URL="http://localhost"
 
+ATHENA_HOST="athena.local"
+
 STRIPE_PRIVATE_API_KEY=""
 STRIPE_PUBLIC_API_KEY=""
 STRIPE_WEBHOOK_SECRET=""
@@ -64,4 +66,11 @@ ELASTICSEARCH_HOST='http://elasticsearch:9200'
 
 
 TENANT_HOSTNAME=ha-local.xyz
-TENANT_DEFAULT_DATABASE=parthenon
+TENANT_DEFAULT_DATABASE=parthenon
+
+AWS_S3_KEY=
+AWS_S3_SECRET=
+AWS_S3_REGION=
+AWS_S3_ENDPOINT=
+AWS_S3_BUCKET=
+UPLOAD_ACCESS_URL=

.gitignore

@@ -16,6 +16,7 @@
 
 /docker/logs/
 /docker/database/data/
+/docker/timescaledb/data/
 .idea
 .php_cs.cache
 
@@ -28,7 +29,6 @@ yarn-error.log
 
 composer.lock
 ###> friendsofphp/php-cs-fixer ###
-/.php-cs-fixer.php
 /.php-cs-fixer.cache
 .php-cs-fixer.cache
 ###< friendsofphp/php-cs-fixer ###

composer.json

@@ -18,6 +18,7 @@
         "monolog/monolog": "^2.8",
         "nyholm/psr7": "^1.5",
         "parthenon/transaction-cloud": "^0.1.0",
+        "phpoffice/phpspreadsheet": "^1.25",
         "psr/http-client": "^1.0",
         "psr/http-message": "^1.0",
         "ramsey/uuid": "^4.5.1",
@@ -32,9 +33,10 @@
         "symfony/expression-language": "^6.1",
         "symfony/flex": "^1.19",
         "symfony/form": "^6.1",
-        "symfony/framework-bundle": "^6.1",
+        "symfony/framework-bundle": "^6.1.7",
         "symfony/mailer": "6.*",
         "symfony/monolog-bundle": "^3.8",
+        "symfony/redis-messenger": "6.1.*",
         "symfony/security-bundle": "^6.1",
         "symfony/translation": "^6.1",
         "symfony/twig-bundle": "^6.1",
@@ -72,7 +74,8 @@
         "symfony/messenger": "^6.1",
         "symfony/phpunit-bridge": "^6.1",
         "toin0u/digitalocean-v2": "^4.5",
-        "wildbit/postmark-php": "^4.0"
+        "wildbit/postmark-php": "^4.0",
+        "symfony/web-profiler-bundle": "^6.1.6"
     },
     "config": {
         "optimize-autoloader": true,
@@ -122,7 +125,7 @@
     "extra": {
         "symfony": {
             "allow-contrib": false,
-            "require": "6.*"
+            "require": "6.1.*"
         }
     }
 }

config/bundles.php

@@ -25,4 +25,5 @@
     Doctrine\Bundle\MongoDBBundle\DoctrineMongoDBBundle::class => ['all' => true],
     Sensio\Bundle\FrameworkExtraBundle\SensioFrameworkExtraBundle::class => ['all' => true],
     Twig\Extra\TwigExtraBundle\TwigExtraBundle::class => ['all' => true],
+    Symfony\Bundle\WebProfilerBundle\WebProfilerBundle::class => ['dev' => true, 'test' => true],
 ];

config/packages/messenger.yaml

@@ -5,18 +5,20 @@ framework:
 
         transports:
             # https://symfony.com/doc/current/messenger.html#transport-configuration
-            # async: '%env(MESSENGER_TRANSPORT_DSN)%'
+            async: '%env(MESSENGER_TRANSPORT_DSN)%'
             # failed: 'doctrine://default?queue_name=failed'
-            # sync: 'sync://'
+            #sync: 'sync://'
 
         routing:
             # Route your messages to the transports
             # 'App\Message\YourMessage': async
+            'Parthenon\Export\BackgroundEmailExportRequest': async
+            'Parthenon\Export\Entity\BackgroundExportRequest': async
 
-# when@test:
-#    framework:
-#        messenger:
-#            transports:
-#                # replace with your transport name here (e.g., my_transport: 'in-memory://')
-#                # For more Messenger testing tools, see https://github.com/zenstruck/messenger-test
-#                async: 'in-memory://'
+when@test:
+    framework:
+        messenger:
+            transports:
+                # replace with your transport name here (e.g., my_transport: 'in-memory://')
+                # For more Messenger testing tools, see https://github.com/zenstruck/messenger-test
+                async: 'in-memory://'

config/packages/pathenon.yaml → config/packages/parthenon.yaml

@@ -55,6 +55,18 @@ parthenon:
       orm_entity_manager: tenant
       default_database: '%env(resolve:TENANT_DEFAULT_DATABASE)%'
   common:
+    uploader:
+      default:
+        provider: s3
+        naming_strategy: random_time
+        url: '%env(resolve:UPLOAD_ACCESS_URL)%'
+        s3:
+          key: '%env(resolve:AWS_S3_KEY)%'
+          secret: '%env(resolve:AWS_S3_SECRET)%'
+          region: '%env(resolve:AWS_S3_REGION)%'
+          endpoint: '%env(resolve:AWS_S3_ENDPOINT)%'
+          bucket_name: '%env(resolve:AWS_S3_BUCKET)%'
+          visibility: public
     elasticsearch:
       connection_type: 'normal'
       hosts: ['%env(resolve:ELASTICSEARCH_HOST)%']
@@ -80,7 +92,6 @@ parthenon:
       provider: sendgrid
       sendgrid:
         api_key: '%env(resolve:SENDGRID_API_KEY)%'
-
   athena:
     enabled: true
   payments:
@@ -133,4 +144,6 @@ parthenon:
           limit:
             team_invite:
               limit: 5
-              description: "Number of users"
+              description: "Number of users"
+  export:
+    enabled: true

config/packages/web_profiler.yaml

@@ -0,0 +1,17 @@
+when@dev:
+    web_profiler:
+        toolbar: true
+        intercept_redirects: false
+
+    framework:
+        profiler:
+            only_exceptions: false
+            collect_serializer_data: true
+
+when@test:
+    web_profiler:
+        toolbar: false
+        intercept_redirects: false
+
+    framework:
+        profiler: { collect: false }

config/routes.yaml

@@ -1,6 +1,3 @@
 #index:
 #    path: /
-#    controller: App\Controller\DefaultController::index
-app_backoffice:
-  resource: .
-  type: athena
+#    controller: App\Controller\DefaultController::index

config/routes/annotations.yaml

@@ -1,3 +1,7 @@
+app_backoffice:
+    resource: .
+    type: athena
+
 controllers:
     resource: ../../src/Controller/
     type: annotation

config/routes/web_profiler.yaml

@@ -0,0 +1,8 @@
+when@dev:
+    web_profiler_wdt:
+        resource: '@WebProfilerBundle/Resources/config/routing/wdt.xml'
+        prefix: /_wdt
+
+    web_profiler_profiler:
+        resource: '@WebProfilerBundle/Resources/config/routing/profiler.xml'
+        prefix: /_profiler

create-public-edition.sh

@@ -7,6 +7,7 @@ rm -Rf ../public-edition/src/AbTesting/*
 rm -Rf ../public-edition/src/Athena/*
 rm -Rf ../public-edition/src/Common/*
 rm -Rf ../public-edition/src/Funnel/*
+rm -Rf ../public-edition/src/Export/*
 rm -Rf ../public-edition/src/Health/*
 rm -Rf ../public-edition/src/Invoice/*
 rm -Rf ../public-edition/src/Notification/*
@@ -60,6 +61,15 @@ cp -R src/Parthenon/Cloud ../public-edition/src/
 cp src/Parthenon/DependencyInjection/Modules/Cloud.php ../public-edition/src/DependencyInjection/Modules/Cloud.php
 cp -R src/Parthenon/Resources/config/services/cloud.xml ../public-edition/src/Resources/config/services/cloud.xml
 
+echo "[x] Update Export"
+cp -R src/Parthenon/Export ../public-edition/src/
+cp src/Parthenon/DependencyInjection/Modules/Export.php ../public-edition/src/DependencyInjection/Modules/Export.php
+cp -R src/Parthenon/Resources/config/doctrine-mapping/Export ../public-edition/src/Resources/config/doctrine-mapping/Export
+cp -R src/Parthenon/Resources/config/services/orm/export.xml ../public-edition/src/Resources/config/services/orm/export.xml
+cp -R src/Parthenon/Resources/config/services/odm/export.xml ../public-edition/src/Resources/config/services/odm/export.xml
+cp -R src/Parthenon/Resources/config/services/export.xml ../public-edition/src/Resources/config/services/export.xml
+cp -R tests/Parthenon/Export ../public-edition/tests/Parthenon/
+
 echo "[x] Update Funnel"
 cp -R src/Parthenon/Funnel ../public-edition/src/
 cp src/Parthenon/DependencyInjection/Modules/Funnel.php ../public-edition/src/DependencyInjection/Modules/Funnel.php

depfile.yaml

@@ -15,6 +15,10 @@ layers:
     collectors:
       - type: directory
         regex: src/Parthenon/Common/*
+  - name: Export
+    collectors:
+      - type: directory
+        regex: src/Parthenon/Export/*
   - name: User
     collectors:
       - type: directory
@@ -54,6 +58,7 @@ ruleset:
     - Athena
   Athena:
     - Common
+    - Export
   User:
     - Common
     - Notification
@@ -66,4 +71,7 @@ ruleset:
     - Common
   Invoice:
     - Common
+  Export:
+    - Common
+    - Notification
   Common:

docker/docker-compose.yml

@@ -43,7 +43,7 @@ services:
       - parthenon
 
   redis:
-    image: redis:4-alpine
+    image: redis:5-alpine
     networks:
       - parthenon
     ports:

migrations/Version20221202162512.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20221202162512 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        // this up() migration is auto-generated, please modify it to your needs
+        $this->addSql('CREATE TABLE parthenon_export_background_export_requests (id UUID NOT NULL, exported_file VARCHAR(255) DEFAULT NULL, exported_file_path VARCHAR(255) DEFAULT NULL, export_format VARCHAR(255) NOT NULL, filename VARCHAR(255) NOT NULL, data_provider_service VARCHAR(255) NOT NULL, parameters TEXT DEFAULT NULL, created_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, updated_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, PRIMARY KEY(id))');
+        $this->addSql('COMMENT ON COLUMN parthenon_export_background_export_requests.id IS \'(DC2Type:uuid)\'');
+        $this->addSql('COMMENT ON COLUMN parthenon_export_background_export_requests.parameters IS \'(DC2Type:array)\'');
+    }
+
+    public function down(Schema $schema): void
+    {
+        // this down() migration is auto-generated, please modify it to your needs
+        $this->addSql('CREATE SCHEMA public');
+        $this->addSql('DROP TABLE parthenon_export_background_export_requests');
+    }
+}

src/Parthenon/Athena/AbstractSection.php

@@ -14,6 +14,7 @@
 
 namespace Parthenon\Athena;
 
+use Parthenon\Athena\Export\NormaliserBuilderInterface;
 use Parthenon\Athena\Filters\ListFilters;
 
 abstract class AbstractSection implements SectionInterface
@@ -82,4 +83,9 @@ public function getButtons(): array
     {
         return [];
     }
+
+    public function buildNormalsier(NormaliserBuilderInterface $builder): NormaliserBuilderInterface
+    {
+        return $builder;
+    }
 }

src/Parthenon/Athena/Compiler/AthenaCompilerPass.php

@@ -16,8 +16,10 @@
 
 use Parthenon\Athena\AccessRightsManagerInterface;
 use Parthenon\Athena\Crud\CrudController;
+use Parthenon\Athena\Export\NormaliserFactoryInterface;
 use Parthenon\Athena\Filters\FilterManager;
 use Parthenon\Athena\ViewTypeManager;
+use Parthenon\Export\Normaliser\NormaliserInterface;
 use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\Definition;
@@ -39,6 +41,10 @@ public function process(ContainerBuilder $container)
         $this->handleViewTypes($container);
     }
 
+    public function handleSectionNormalisers(ContainerBuilder $containerBuilder): void
+    {
+    }
+
     private function handleViewTypes(ContainerBuilder $container): void
     {
         $viewTypeManager = $container->getDefinition('parthenon.athena.view_type_manager');
@@ -150,6 +156,14 @@ private function processSectionDefinitions(ContainerBuilder $container, array $d
             $controllerDefinition->setAutowired(true);
 
             $container->setDefinition('athena_controller_'.$servicePart, $controllerDefinition);
+
+            $normaliserDefinition = new Definition();
+            $normaliserDefinition->setClass(NormaliserInterface::class);
+            $normaliserDefinition->setFactory([new Reference(NormaliserFactoryInterface::class), 'build']);
+            $normaliserDefinition->setArgument('$section', new Reference($name));
+            $normaliserDefinition->addTag('parthenon.export.normaliser');
+
+            $container->setDefinition('athena_export_normaliser_'.$servicePart, $normaliserDefinition);
         }
     }
 

src/Parthenon/Athena/Controller/ExportController.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * Copyright Iain Cambridge 2020-2022.
+ *
+ * Use of this software is governed by the Business Source License included in the LICENSE file and at https://getparthenon.com/docs/next/license.
+ *
+ * Change Date: TBD ( 3 years after 2.1.0 release )
+ *
+ * On the date above, in accordance with the Business Source License, use of this software will be governed by the open source license specified in the LICENSE file.
+ */
+
+namespace Parthenon\Athena\Controller;
+
+use Parthenon\Export\Entity\BackgroundExportRequest;
+use Parthenon\Export\Repository\BackgroundExportRequestRepositoryInterface;
+use Psr\Log\LoggerInterface;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Template;
+use Symfony\Component\HttpFoundation\RedirectResponse;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Twig\Environment;
+
+class ExportController
+{
+    #[Template('@Parthenon/athena/crud/export_background_email.html.twig')]
+    public function emailWaiting(LoggerInterface $logger)
+    {
+        $logger->info("An Athena user has been told they'll receive an email");
+
+        return [];
+    }
+
+    public function downloadWaiting(Request $request, LoggerInterface $logger, BackgroundExportRequestRepositoryInterface $backgroundExportRequestRepository, Environment $twig)
+    {
+        $logger->info('An Athena user has checked the process of an export');
+
+        $id = $request->get('id');
+
+        /** @var BackgroundExportRequest $backgroundExportRequest */
+        $backgroundExportRequest = $backgroundExportRequestRepository->findById($id);
+
+        $downloadUrl = $backgroundExportRequest->getExportedFilePath();
+
+        if ($downloadUrl) {
+            return new RedirectResponse($downloadUrl);
+        } else {
+            return new Response($twig->render('@Parthenon/athena/crud/export_background_download.html.twig'));
+        }
+    }
+}