Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion docs/reference/tasks/json_stream_reader_task.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ Underlying method are [SplFileObject::fgets](https://www.php.net/manual/fr/splfi
Options
-------

none
| Code | Type | Required | Default | Description |
|-------------------------|-----------------|:--------:|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `spl_file_object_flags` | `array`, `null` | | `\SplFileObject::DROP_NEW_LINE \SplFileObject::READ_AHEAD \SplFileObject::SKIP_EMPTY` | Flags to pass to `SplFileObject` constructor, can be empty.<br/>See [PHP documentation](https://www.php.net/manual/en/splfileobject.construct.php) for more information on available flags. |
| `json_flags` | `array`, `null` | | `\JSON_THROW_ON_ERROR` | Flags to pass to `json_encode` function, can be empty.<br/>See [PHP documentation](https://www.php.net/manual/en/function.json-encode.php) for more information on available flags. |


Example
-------
Expand All @@ -38,6 +42,10 @@ entry:
file_path: '%kernel.project_dir%/var/data/json_stream_reader.json'
read:
service: '@CleverAge\ProcessBundle\Task\File\JsonStream\JsonStreamReaderTask'
options:
spl_file_object_flags: []
json_flags:
- !php/const JSON_ERROR_NONE
```


12 changes: 9 additions & 3 deletions docs/reference/tasks/json_stream_writer_task.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ Possible outputs
Options
-------

| Code | Type | Required | Default | Description |
|-------------|----------|:--------:|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `file_path` | `string` | **X** | | Path of the file to write to (relative to symfony root or absolute).<br/>It can also take placeholders (`{date}`, `{date_time}`, `{timestamp}` `{unique_token}`) to insert data into the filename |
| Code | Type | Required | Default | Description |
|-------------------------|-----------------|:--------:|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `file_path` | `string` | **X** | | Path of the file to write to (relative to symfony root or absolute).<br/>It can also take placeholders (`{date}`, `{date_time}`, `{timestamp}` `{unique_token}`) to insert data into the filename |
| `spl_file_object_flags` | `array`, `null` | | `\SplFileObject::DROP_NEW_LINE \SplFileObject::READ_AHEAD \SplFileObject::SKIP_EMPTY` | Flags to pass to `SplFileObject` constructor, can be empty.<br/>See [PHP documentation](https://www.php.net/manual/en/splfileobject.construct.php) for more information on available flags. |
| `json_flags` | `array`, `null` | | `\JSON_THROW_ON_ERROR` | Flags to pass to `json_encode` function, can be empty.<br/>See [PHP documentation](https://www.php.net/manual/en/function.json-encode.php) for more information on available flags. |

Example
----------------
Expand All @@ -50,4 +52,8 @@ write:
service: '@CleverAge\ProcessBundle\Task\File\JsonStream\JsonStreamWriterTask'
options:
file_path: '%kernel.project_dir%/var/data/json_stream_writer_{date_time}.csv'
spl_file_object_flags: []
json_flags:
- !php/const JSON_PRETTY_PRINT
- !php/const JSON_UNESCAPED_SLASHES
```
24 changes: 19 additions & 5 deletions src/Filesystem/JsonStreamFile.php
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,30 @@ class JsonStreamFile implements FileStreamInterface, WritableFileInterface
{
protected \SplFileObject $file;

private readonly int $jsonFlags;

protected ?int $lineCount = null;

protected int $lineNumber = 1;

public function __construct(string $filename, string $mode = 'rb')
{
public function __construct(
string $filename,
string $mode = 'rb',
?array $splFileObjectFlags = null,
?array $jsonFlags = null,
) {
$this->file = new \SplFileObject($filename, $mode);

// Useful to skip empty trailing lines (doesn't work well on PHP 8, see readLine() code)
$this->file->setFlags(\SplFileObject::DROP_NEW_LINE | \SplFileObject::READ_AHEAD | \SplFileObject::SKIP_EMPTY);
$this->file->setFlags(null !== $splFileObjectFlags
? array_sum($splFileObjectFlags)
: \SplFileObject::DROP_NEW_LINE | \SplFileObject::READ_AHEAD | \SplFileObject::SKIP_EMPTY
);

$this->jsonFlags = null !== $jsonFlags
? array_sum($jsonFlags)
: \JSON_THROW_ON_ERROR
;
}

/**
Expand Down Expand Up @@ -78,12 +92,12 @@ public function readLine(?int $length = null): ?array
}
++$this->lineNumber;

return json_decode($rawLine, true, 512, \JSON_THROW_ON_ERROR);
return json_decode($rawLine, true, 512, $this->jsonFlags);
}

public function writeLine(array $fields): int
{
$this->file->fwrite(json_encode($fields, \JSON_THROW_ON_ERROR).\PHP_EOL);
$this->file->fwrite(json_encode($fields, $this->jsonFlags).\PHP_EOL);
++$this->lineNumber;

return $this->lineNumber;
Expand Down
22 changes: 20 additions & 2 deletions src/Task/File/JsonStream/JsonStreamReaderTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,25 @@
namespace CleverAge\ProcessBundle\Task\File\JsonStream;

use CleverAge\ProcessBundle\Filesystem\JsonStreamFile;
use CleverAge\ProcessBundle\Model\AbstractConfigurableTask;
use CleverAge\ProcessBundle\Model\IterableTaskInterface;
use CleverAge\ProcessBundle\Model\ProcessState;
use Symfony\Component\OptionsResolver\OptionsResolver;

class JsonStreamReaderTask implements IterableTaskInterface
class JsonStreamReaderTask extends AbstractConfigurableTask implements IterableTaskInterface
{
protected ?JsonStreamFile $file = null;

public function execute(ProcessState $state): void
{
if (!$this->file instanceof JsonStreamFile) {
$this->file = new JsonStreamFile($this->getFilePath($state), 'rb');
$options = $this->getOptions($state);
$this->file = new JsonStreamFile(
$this->getFilePath($state),
'rb',
$options['spl_file_object_flags'],
$options['json_flags'],
);
}

$line = $this->file->readLine();
Expand All @@ -49,4 +57,14 @@ protected function getFilePath(ProcessState $state): string
{
return $state->getInput();
}

protected function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefaults([
'spl_file_object_flags' => null,
'json_flags' => null,
]);
$resolver->setAllowedTypes('spl_file_object_flags', ['array', 'null']);
$resolver->setAllowedTypes('json_flags', ['array', 'null']);
}
}
15 changes: 13 additions & 2 deletions src/Task/File/JsonStream/JsonStreamWriterTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,14 @@ class JsonStreamWriterTask extends AbstractConfigurableTask implements BlockingT

public function execute(ProcessState $state): void
{
$options = $this->getOptions($state);
if (!$this->file instanceof JsonStreamFile) {
$this->file = new JsonStreamFile($options['file_path'], 'wb');
$options = $this->getOptions($state);
$this->file = new JsonStreamFile(
$options['file_path'],
'wb',
$options['spl_file_object_flags'],
$options['json_flags'],
);
}

$input = $state->getInput();
Expand Down Expand Up @@ -60,5 +65,11 @@ protected function configureOptions(OptionsResolver $resolver): void
]
)
);
$resolver->setDefaults([
'spl_file_object_flags' => null,
'json_flags' => null,
]);
$resolver->setAllowedTypes('spl_file_object_flags', ['array', 'null']);
$resolver->setAllowedTypes('json_flags', ['array', 'null']);
}
}