Skip to content

Fix up metadata normalizing in legacy serializer #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 19, 2024
Merged

Fix up metadata normalizing in legacy serializer #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
981f73d
Fix up metadata normalizing in legacy serializer
AdamVyborny Jun 19, 2024
71004c7
Remove legacy badges
AdamVyborny Jun 19, 2024
f74f88f
Update readme with new flag
AdamVyborny Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
# Keboola PHP Component

[![Build Status](https://travis-ci.com/keboola/php-component.svg?branch=master)](https://travis-ci.com/keboola/php-component)
[![Code Climate](https://codeclimate.com/github/keboola/php-component/badges/gpa.svg)](https://codeclimate.com/github/keboola/php-component)

General library for php component running in KBC. The library provides function related to [Docker Runner](https://github.com/keboola/docker-bundle).

## Installation
Expand Down Expand Up @@ -48,7 +45,8 @@ class Component extends \Keboola\Component\BaseComponent
'data.csv',
(new OutTableManifestOptions())
->setPrimaryKeyColumns(['id'])
->setDestination('out.report')
->setDestination('out.report'),
true // legacy manifest format flag
);
}

Expand Down
17 changes: 13 additions & 4 deletions src/Manifest/ManifestManager/Options/OutTable/Serializer/LegacyManifestNormalizer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ private function normalizeSchema(ManifestOptions $object, array &$data): void
$data['primary_key'][] = $schema->getName();
}

$columnMetadata[] = ['key' => 'KBC.datatype.nullable', 'value' => $schema->isNullable()];
if ($schema->getDescription() !== null) {
$columnMetadata[] = ['key' => 'KBC.description', 'value' => $schema->getDescription()];
}

$this->normalizeDataTypes($schema, $columnMetadata);
$this->normalizeColumnMetadata($schema, $columnMetadata);

Expand All @@ -79,9 +84,13 @@ private function normalizeDataTypes(ManifestOptionsSchema $schema, array &$colum
{
if ($schema->getDataType() !== null) {
foreach ($schema->getDataType() as $backend => $typeInfo) {
foreach ($typeInfo as $key => $value) {
$metaKey = ($backend === 'base' ? 'KBC.datatype.basetype' : 'KBC.datatype.' . $key);
$columnMetadata[] = ['key' => $metaKey, 'value' => $value];
$typeMetaKey = ($backend === 'base' ? 'KBC.datatype.basetype' : 'KBC.datatype.type');
$columnMetadata[] = ['key' => $typeMetaKey, 'value' => $typeInfo->getType()];
if ($typeInfo->getLength() !== null) {
$columnMetadata[] = ['key' => 'KBC.datatype.length', 'value' => $typeInfo->getLength()];
}
if ($typeInfo->getDefault() !== null) {
$columnMetadata[] = ['key' => 'KBC.datatype.default', 'value' => $typeInfo->getDefault()];
}
}
}
Expand Down Expand Up @@ -176,7 +185,7 @@ private function setSchema(
$description = null;

foreach ($columnMetadata as $meta) {
if (strpos($meta['key'], 'KBC.datatype.') === 0 && $meta['key'] !== 'KBC.datatype.nullable') {
if (str_starts_with($meta['key'], 'KBC.datatype.') && $meta['key'] !== 'KBC.datatype.nullable') {
$this->setDataType($meta, $dataTypes, $metadataBackend);
} else {
$this->setMetadata($meta, $metadata, $description, $primaryKey, $isNullable);
Expand Down
106 changes: 106 additions & 0 deletions tests/Manifest/ManifestManagerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,44 @@ public function testWillLoadTableManifest(): void
'id',
'number',
],
'column_metadata' => [
'id' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'number' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'name' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'description' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'created_at' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'updated_at' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
],
];

$this->assertSame($expectedManifest, $manager->getTableManifest('people.csv')->toArray());
Expand Down Expand Up @@ -140,6 +178,26 @@ public function testWillLoadTableManifestWithoutCsv(): void
'id',
'number',
],
'column_metadata' => [
'id' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'number' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
'name' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
],
],
];

$this->assertSame($expectedManifest, $manager->getTableManifest('products')->toArray());
Expand Down Expand Up @@ -429,11 +487,59 @@ public function testConvertNewDataTypesManifestToLegacyArray(): void
'columns' => ['id', 'number', 'other_column'],
'column_metadata' => [
'id' => [
[
'key' => 'KBC.datatype.nullable',
'value' => false,
],
[
'key' => 'KBC.description',
'value' => 'This is a primary key',
],
[
'key' => 'KBC.datatype.basetype',
'value' => 'INTEGER',
],
[
'key' => 'KBC.datatype.length',
'value' => '11',
],
[
'key' => 'KBC.datatype.default',
'value' => '123',
],
[
'key' => 'yet.another.key',
'value' => 'Some other value',
],
],
'number' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
[
'key' => 'KBC.datatype.basetype',
'value' => 'VARCHAR',
],
[
'key' => 'KBC.datatype.length',
'value' => '255',
],
],
'other_column' => [
[
'key' => 'KBC.datatype.nullable',
'value' => true,
],
[
'key' => 'KBC.datatype.basetype',
'value' => 'VARCHAR',
],
[
'key' => 'KBC.datatype.length',
'value' => '255',
],
],
],
];

Expand Down