diff --git a/docs/dao.html b/docs/dao.html index 4a64782..258413d 100644 --- a/docs/dao.html +++ b/docs/dao.html @@ -2046,6 +2046,11 @@

Args

"columns", "metadata", "column_metadata", + "rows_count", + "data_size_bytes", + "is_alias", + "attributes", + "indexed_columns" ] OUTPUT_MANIFEST_ATTRIBUTES = [ @@ -2158,6 +2163,8 @@

Args

self._rows_count = kwargs.get('rows_count') self._data_size_bytes = kwargs.get('data_size_bytes') self._is_alias = kwargs.get('is_alias') + self._indexed_columns = kwargs.get('indexed_columns') + self._attributes = kwargs.get('attributes') self.stage = stage self.has_header = has_header or self._has_header_in_file() @@ -2410,7 +2417,9 @@

Args

rows_count=manifest.get('rows_count'), data_size_bytes=manifest.get('data_size_bytes'), is_alias=manifest.get('is_alias'), - force_legacy_mode=force_legacy_mode + force_legacy_mode=force_legacy_mode, + indexed_columns=manifest.get('indexed_columns'), + attributes=manifest.get('attributes') ) return table_def @@ -2487,6 +2496,8 @@

Args

'rows_count': self._rows_count, 'data_size_bytes': self._data_size_bytes, 'is_alias': self._is_alias, + 'indexed_columns': self._indexed_columns, + 'attributes': self._attributes, 'destination': self.destination, 'incremental': self.incremental, diff --git a/setup.py b/setup.py index 90afaa3..397759f 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setuptools.setup( name="keboola.component", - version="1.6.3", + version="1.6.4", author="Keboola KDS Team", project_urls=project_urls, setup_requires=['pytest-runner', 'flake8'], diff --git a/src/keboola/component/dao.py b/src/keboola/component/dao.py index 3005867..b2ddc76 100644 --- a/src/keboola/component/dao.py +++ b/src/keboola/component/dao.py @@ -744,6 +744,11 @@ class TableDefinition(IODefinition): "columns", "metadata", "column_metadata", + "rows_count", + "data_size_bytes", + "is_alias", + "attributes", + "indexed_columns" ] OUTPUT_MANIFEST_ATTRIBUTES = [ @@ -856,6 +861,8 @@ def __init__(self, name: str, self._rows_count = kwargs.get('rows_count') self._data_size_bytes = kwargs.get('data_size_bytes') self._is_alias = kwargs.get('is_alias') + self._indexed_columns = kwargs.get('indexed_columns') + self._attributes = kwargs.get('attributes') self.stage = stage self.has_header = has_header or self._has_header_in_file() @@ -1108,7 +1115,9 @@ def build_from_manifest(cls, rows_count=manifest.get('rows_count'), data_size_bytes=manifest.get('data_size_bytes'), is_alias=manifest.get('is_alias'), - force_legacy_mode=force_legacy_mode + force_legacy_mode=force_legacy_mode, + indexed_columns=manifest.get('indexed_columns'), + attributes=manifest.get('attributes') ) return table_def @@ -1185,6 +1194,8 @@ def _filter_attributes_by_manifest_type(self, manifest_type: Literal["in", "out" 'rows_count': self._rows_count, 'data_size_bytes': self._data_size_bytes, 'is_alias': self._is_alias, + 'indexed_columns': self._indexed_columns, + 'attributes': self._attributes, 'destination': self.destination, 'incremental': self.incremental, diff --git a/tests/data_examples/data_full_input_manifest/config.json b/tests/data_examples/data_full_input_manifest/config.json new file mode 100644 index 0000000..5dc9795 --- /dev/null +++ b/tests/data_examples/data_full_input_manifest/config.json @@ -0,0 +1,31 @@ +{ + "storage": { + "input": { + "files": [], + "tables": [ + { + "source": "in.c-main.test", + "destination": "sample.csv", + "columns": [], + "where_values": [], + "where_operator": "eq", + "column_types": [ + { + "source": "Sales", + "type": "VARCHAR", + "destination": "id", + "length": "255", + "nullable": false, + "convert_empty_values_to_null": false + } + ] + } + ] + } + + }, + "parameters": { + + } + +} diff --git a/tests/data_examples/data_full_input_manifest/in/state.json b/tests/data_examples/data_full_input_manifest/in/state.json new file mode 100644 index 0000000..078a640 --- /dev/null +++ b/tests/data_examples/data_full_input_manifest/in/state.json @@ -0,0 +1,3 @@ +{ + "test_state": 1234 +} \ No newline at end of file diff --git a/tests/data_examples/data_full_input_manifest/in/tables/sample.csv b/tests/data_examples/data_full_input_manifest/in/tables/sample.csv new file mode 100644 index 0000000..d58791a --- /dev/null +++ b/tests/data_examples/data_full_input_manifest/in/tables/sample.csv @@ -0,0 +1,401 @@ +"x","Sales","CompPrice","Income","Advertising","Population","Price","ShelveLoc","Age","Education","Urban","US","High" +"1","9.5","138","73","11","276","120","Bad","42","17","Yes","Yes","Yes" +"2","11.22","111","48","16","260","83","Good","65","10","Yes","Yes","Yes" +"3","10.06","113","35","10","269","80","Medium","59","12","Yes","Yes","Yes" +"4","7.4","117","100","4","466","97","Medium","55","14","Yes","Yes","No" +"5","4.15","141","64","3","340","128","Bad","38","13","Yes","No","No" +"6","10.81","124","113","13","501","72","Bad","78","16","No","Yes","Yes" +"7","6.63","115","105","0","45","108","Medium","71","15","Yes","No","No" +"8","11.85","136","81","15","425","120","Good","67","10","Yes","Yes","Yes" +"9","6.54","132","110","0","108","124","Medium","76","10","No","No","No" +"10","4.69","132","113","0","131","124","Medium","76","17","No","Yes","No" +"11","9.01","121","78","9","150","100","Bad","26","10","No","Yes","Yes" +"12","11.96","117","94","4","503","94","Good","50","13","Yes","Yes","Yes" +"13","3.98","122","35","2","393","136","Medium","62","18","Yes","No","No" +"14","10.96","115","28","11","29","86","Good","53","18","Yes","Yes","Yes" +"15","11.17","107","117","11","148","118","Good","52","18","Yes","Yes","Yes" +"16","8.71","149","95","5","400","144","Medium","76","18","No","No","Yes" +"17","7.58","118","32","0","284","110","Good","63","13","Yes","No","No" +"18","12.29","147","74","13","251","131","Good","52","10","Yes","Yes","Yes" +"19","13.91","110","110","0","408","68","Good","46","17","No","Yes","Yes" +"20","8.73","129","76","16","58","121","Medium","69","12","Yes","Yes","Yes" +"21","6.41","125","90","2","367","131","Medium","35","18","Yes","Yes","No" +"22","12.13","134","29","12","239","109","Good","62","18","No","Yes","Yes" +"23","5.08","128","46","6","497","138","Medium","42","13","Yes","No","No" +"24","5.87","121","31","0","292","109","Medium","79","10","Yes","No","No" +"25","10.14","145","119","16","294","113","Bad","42","12","Yes","Yes","Yes" +"26","14.9","139","32","0","176","82","Good","54","11","No","No","Yes" +"27","8.33","107","115","11","496","131","Good","50","11","No","Yes","Yes" +"28","5.27","98","118","0","19","107","Medium","64","17","Yes","No","No" +"29","2.99","103","74","0","359","97","Bad","55","11","Yes","Yes","No" +"30","7.81","104","99","15","226","102","Bad","58","17","Yes","Yes","No" +"31","13.55","125","94","0","447","89","Good","30","12","Yes","No","Yes" +"32","8.25","136","58","16","241","131","Medium","44","18","Yes","Yes","Yes" +"33","6.2","107","32","12","236","137","Good","64","10","No","Yes","No" +"34","8.77","114","38","13","317","128","Good","50","16","Yes","Yes","Yes" +"35","2.67","115","54","0","406","128","Medium","42","17","Yes","Yes","No" +"36","11.07","131","84","11","29","96","Medium","44","17","No","Yes","Yes" +"37","8.89","122","76","0","270","100","Good","60","18","No","No","Yes" +"38","4.95","121","41","5","412","110","Medium","54","10","Yes","Yes","No" +"39","6.59","109","73","0","454","102","Medium","65","15","Yes","No","No" +"40","3.24","130","60","0","144","138","Bad","38","10","No","No","No" +"41","2.07","119","98","0","18","126","Bad","73","17","No","No","No" +"42","7.96","157","53","0","403","124","Bad","58","16","Yes","No","No" +"43","10.43","77","69","0","25","24","Medium","50","18","Yes","No","Yes" +"44","4.12","123","42","11","16","134","Medium","59","13","Yes","Yes","No" +"45","4.16","85","79","6","325","95","Medium","69","13","Yes","Yes","No" +"46","4.56","141","63","0","168","135","Bad","44","12","Yes","Yes","No" +"47","12.44","127","90","14","16","70","Medium","48","15","No","Yes","Yes" +"48","4.38","126","98","0","173","108","Bad","55","16","Yes","No","No" +"49","3.91","116","52","0","349","98","Bad","69","18","Yes","No","No" +"50","10.61","157","93","0","51","149","Good","32","17","Yes","No","Yes" +"51","1.42","99","32","18","341","108","Bad","80","16","Yes","Yes","No" +"52","4.42","121","90","0","150","108","Bad","75","16","Yes","No","No" +"53","7.91","153","40","3","112","129","Bad","39","18","Yes","Yes","No" +"54","6.92","109","64","13","39","119","Medium","61","17","Yes","Yes","No" +"55","4.9","134","103","13","25","144","Medium","76","17","No","Yes","No" +"56","6.85","143","81","5","60","154","Medium","61","18","Yes","Yes","No" +"57","11.91","133","82","0","54","84","Medium","50","17","Yes","No","Yes" +"58","0.91","93","91","0","22","117","Bad","75","11","Yes","No","No" +"59","5.42","103","93","15","188","103","Bad","74","16","Yes","Yes","No" +"60","5.21","118","71","4","148","114","Medium","80","13","Yes","No","No" +"61","8.32","122","102","19","469","123","Bad","29","13","Yes","Yes","Yes" +"62","7.32","105","32","0","358","107","Medium","26","13","No","No","No" +"63","1.82","139","45","0","146","133","Bad","77","17","Yes","Yes","No" +"64","8.47","119","88","10","170","101","Medium","61","13","Yes","Yes","Yes" +"65","7.8","100","67","12","184","104","Medium","32","16","No","Yes","No" +"66","4.9","122","26","0","197","128","Medium","55","13","No","No","No" +"67","8.85","127","92","0","508","91","Medium","56","18","Yes","No","Yes" +"68","9.01","126","61","14","152","115","Medium","47","16","Yes","Yes","Yes" +"69","13.39","149","69","20","366","134","Good","60","13","Yes","Yes","Yes" +"70","7.99","127","59","0","339","99","Medium","65","12","Yes","No","No" +"71","9.46","89","81","15","237","99","Good","74","12","Yes","Yes","Yes" +"72","6.5","148","51","16","148","150","Medium","58","17","No","Yes","No" +"73","5.52","115","45","0","432","116","Medium","25","15","Yes","No","No" +"74","12.61","118","90","10","54","104","Good","31","11","No","Yes","Yes" +"75","6.2","150","68","5","125","136","Medium","64","13","No","Yes","No" +"76","8.55","88","111","23","480","92","Bad","36","16","No","Yes","Yes" +"77","10.64","102","87","10","346","70","Medium","64","15","Yes","Yes","Yes" +"78","7.7","118","71","12","44","89","Medium","67","18","No","Yes","No" +"79","4.43","134","48","1","139","145","Medium","65","12","Yes","Yes","No" +"80","9.14","134","67","0","286","90","Bad","41","13","Yes","No","Yes" +"81","8.01","113","100","16","353","79","Bad","68","11","Yes","Yes","Yes" +"82","7.52","116","72","0","237","128","Good","70","13","Yes","No","No" +"83","11.62","151","83","4","325","139","Good","28","17","Yes","Yes","Yes" +"84","4.42","109","36","7","468","94","Bad","56","11","Yes","Yes","No" +"85","2.23","111","25","0","52","121","Bad","43","18","No","No","No" +"86","8.47","125","103","0","304","112","Medium","49","13","No","No","Yes" +"87","8.7","150","84","9","432","134","Medium","64","15","Yes","No","Yes" +"88","11.7","131","67","7","272","126","Good","54","16","No","Yes","Yes" +"89","6.56","117","42","7","144","111","Medium","62","10","Yes","Yes","No" +"90","7.95","128","66","3","493","119","Medium","45","16","No","No","No" +"91","5.33","115","22","0","491","103","Medium","64","11","No","No","No" +"92","4.81","97","46","11","267","107","Medium","80","15","Yes","Yes","No" +"93","4.53","114","113","0","97","125","Medium","29","12","Yes","No","No" +"94","8.86","145","30","0","67","104","Medium","55","17","Yes","No","Yes" +"95","8.39","115","97","5","134","84","Bad","55","11","Yes","Yes","Yes" +"96","5.58","134","25","10","237","148","Medium","59","13","Yes","Yes","No" +"97","9.48","147","42","10","407","132","Good","73","16","No","Yes","Yes" +"98","7.45","161","82","5","287","129","Bad","33","16","Yes","Yes","No" +"99","12.49","122","77","24","382","127","Good","36","16","No","Yes","Yes" +"100","4.88","121","47","3","220","107","Bad","56","16","No","Yes","No" +"101","4.11","113","69","11","94","106","Medium","76","12","No","Yes","No" +"102","6.2","128","93","0","89","118","Medium","34","18","Yes","No","No" +"103","5.3","113","22","0","57","97","Medium","65","16","No","No","No" +"104","5.07","123","91","0","334","96","Bad","78","17","Yes","Yes","No" +"105","4.62","121","96","0","472","138","Medium","51","12","Yes","No","No" +"106","5.55","104","100","8","398","97","Medium","61","11","Yes","Yes","No" +"107","0.16","102","33","0","217","139","Medium","70","18","No","No","No" +"108","8.55","134","107","0","104","108","Medium","60","12","Yes","No","Yes" +"109","3.47","107","79","2","488","103","Bad","65","16","Yes","No","No" +"110","8.98","115","65","0","217","90","Medium","60","17","No","No","Yes" +"111","9","128","62","7","125","116","Medium","43","14","Yes","Yes","Yes" +"112","6.62","132","118","12","272","151","Medium","43","14","Yes","Yes","No" +"113","6.67","116","99","5","298","125","Good","62","12","Yes","Yes","No" +"114","6.01","131","29","11","335","127","Bad","33","12","Yes","Yes","No" +"115","9.31","122","87","9","17","106","Medium","65","13","Yes","Yes","Yes" +"116","8.54","139","35","0","95","129","Medium","42","13","Yes","No","Yes" +"117","5.08","135","75","0","202","128","Medium","80","10","No","No","No" +"118","8.8","145","53","0","507","119","Medium","41","12","Yes","No","Yes" +"119","7.57","112","88","2","243","99","Medium","62","11","Yes","Yes","No" +"120","7.37","130","94","8","137","128","Medium","64","12","Yes","Yes","No" +"121","6.87","128","105","11","249","131","Medium","63","13","Yes","Yes","No" +"122","11.67","125","89","10","380","87","Bad","28","10","Yes","Yes","Yes" +"123","6.88","119","100","5","45","108","Medium","75","10","Yes","Yes","No" +"124","8.19","127","103","0","125","155","Good","29","15","No","Yes","Yes" +"125","8.87","131","113","0","181","120","Good","63","14","Yes","No","Yes" +"126","9.34","89","78","0","181","49","Medium","43","15","No","No","Yes" +"127","11.27","153","68","2","60","133","Good","59","16","Yes","Yes","Yes" +"128","6.52","125","48","3","192","116","Medium","51","14","Yes","Yes","No" +"129","4.96","133","100","3","350","126","Bad","55","13","Yes","Yes","No" +"130","4.47","143","120","7","279","147","Bad","40","10","No","Yes","No" +"131","8.41","94","84","13","497","77","Medium","51","12","Yes","Yes","Yes" +"132","6.5","108","69","3","208","94","Medium","77","16","Yes","No","No" +"133","9.54","125","87","9","232","136","Good","72","10","Yes","Yes","Yes" +"134","7.62","132","98","2","265","97","Bad","62","12","Yes","Yes","No" +"135","3.67","132","31","0","327","131","Medium","76","16","Yes","No","No" +"136","6.44","96","94","14","384","120","Medium","36","18","No","Yes","No" +"137","5.17","131","75","0","10","120","Bad","31","18","No","No","No" +"138","6.52","128","42","0","436","118","Medium","80","11","Yes","No","No" +"139","10.27","125","103","12","371","109","Medium","44","10","Yes","Yes","Yes" +"140","12.3","146","62","10","310","94","Medium","30","13","No","Yes","Yes" +"141","6.03","133","60","10","277","129","Medium","45","18","Yes","Yes","No" +"142","6.53","140","42","0","331","131","Bad","28","15","Yes","No","No" +"143","7.44","124","84","0","300","104","Medium","77","15","Yes","No","No" +"144","0.53","122","88","7","36","159","Bad","28","17","Yes","Yes","No" +"145","9.09","132","68","0","264","123","Good","34","11","No","No","Yes" +"146","8.77","144","63","11","27","117","Medium","47","17","Yes","Yes","Yes" +"147","3.9","114","83","0","412","131","Bad","39","14","Yes","No","No" +"148","10.51","140","54","9","402","119","Good","41","16","No","Yes","Yes" +"149","7.56","110","119","0","384","97","Medium","72","14","No","Yes","No" +"150","11.48","121","120","13","140","87","Medium","56","11","Yes","Yes","Yes" +"151","10.49","122","84","8","176","114","Good","57","10","No","Yes","Yes" +"152","10.77","111","58","17","407","103","Good","75","17","No","Yes","Yes" +"153","7.64","128","78","0","341","128","Good","45","13","No","No","No" +"154","5.93","150","36","7","488","150","Medium","25","17","No","Yes","No" +"155","6.89","129","69","10","289","110","Medium","50","16","No","Yes","No" +"156","7.71","98","72","0","59","69","Medium","65","16","Yes","No","No" +"157","7.49","146","34","0","220","157","Good","51","16","Yes","No","No" +"158","10.21","121","58","8","249","90","Medium","48","13","No","Yes","Yes" +"159","12.53","142","90","1","189","112","Good","39","10","No","Yes","Yes" +"160","9.32","119","60","0","372","70","Bad","30","18","No","No","Yes" +"161","4.67","111","28","0","486","111","Medium","29","12","No","No","No" +"162","2.93","143","21","5","81","160","Medium","67","12","No","Yes","No" +"163","3.63","122","74","0","424","149","Medium","51","13","Yes","No","No" +"164","5.68","130","64","0","40","106","Bad","39","17","No","No","No" +"165","8.22","148","64","0","58","141","Medium","27","13","No","Yes","Yes" +"166","0.37","147","58","7","100","191","Bad","27","15","Yes","Yes","No" +"167","6.71","119","67","17","151","137","Medium","55","11","Yes","Yes","No" +"168","6.71","106","73","0","216","93","Medium","60","13","Yes","No","No" +"169","7.3","129","89","0","425","117","Medium","45","10","Yes","No","No" +"170","11.48","104","41","15","492","77","Good","73","18","Yes","Yes","Yes" +"171","8.01","128","39","12","356","118","Medium","71","10","Yes","Yes","Yes" +"172","12.49","93","106","12","416","55","Medium","75","15","Yes","Yes","Yes" +"173","9.03","104","102","13","123","110","Good","35","16","Yes","Yes","Yes" +"174","6.38","135","91","5","207","128","Medium","66","18","Yes","Yes","No" +"175","0","139","24","0","358","185","Medium","79","15","No","No","No" +"176","7.54","115","89","0","38","122","Medium","25","12","Yes","No","No" +"177","5.61","138","107","9","480","154","Medium","47","11","No","Yes","No" +"178","10.48","138","72","0","148","94","Medium","27","17","Yes","Yes","Yes" +"179","10.66","104","71","14","89","81","Medium","25","14","No","Yes","Yes" +"180","7.78","144","25","3","70","116","Medium","77","18","Yes","Yes","No" +"181","4.94","137","112","15","434","149","Bad","66","13","Yes","Yes","No" +"182","7.43","121","83","0","79","91","Medium","68","11","Yes","No","No" +"183","4.74","137","60","4","230","140","Bad","25","13","Yes","No","No" +"184","5.32","118","74","6","426","102","Medium","80","18","Yes","Yes","No" +"185","9.95","132","33","7","35","97","Medium","60","11","No","Yes","Yes" +"186","10.07","130","100","11","449","107","Medium","64","10","Yes","Yes","Yes" +"187","8.68","120","51","0","93","86","Medium","46","17","No","No","Yes" +"188","6.03","117","32","0","142","96","Bad","62","17","Yes","No","No" +"189","8.07","116","37","0","426","90","Medium","76","15","Yes","No","Yes" +"190","12.11","118","117","18","509","104","Medium","26","15","No","Yes","Yes" +"191","8.79","130","37","13","297","101","Medium","37","13","No","Yes","Yes" +"192","6.67","156","42","13","170","173","Good","74","14","Yes","Yes","No" +"193","7.56","108","26","0","408","93","Medium","56","14","No","No","No" +"194","13.28","139","70","7","71","96","Good","61","10","Yes","Yes","Yes" +"195","7.23","112","98","18","481","128","Medium","45","11","Yes","Yes","No" +"196","4.19","117","93","4","420","112","Bad","66","11","Yes","Yes","No" +"197","4.1","130","28","6","410","133","Bad","72","16","Yes","Yes","No" +"198","2.52","124","61","0","333","138","Medium","76","16","Yes","No","No" +"199","3.62","112","80","5","500","128","Medium","69","10","Yes","Yes","No" +"200","6.42","122","88","5","335","126","Medium","64","14","Yes","Yes","No" +"201","5.56","144","92","0","349","146","Medium","62","12","No","No","No" +"202","5.94","138","83","0","139","134","Medium","54","18","Yes","No","No" +"203","4.1","121","78","4","413","130","Bad","46","10","No","Yes","No" +"204","2.05","131","82","0","132","157","Bad","25","14","Yes","No","No" +"205","8.74","155","80","0","237","124","Medium","37","14","Yes","No","Yes" +"206","5.68","113","22","1","317","132","Medium","28","12","Yes","No","No" +"207","4.97","162","67","0","27","160","Medium","77","17","Yes","Yes","No" +"208","8.19","111","105","0","466","97","Bad","61","10","No","No","Yes" +"209","7.78","86","54","0","497","64","Bad","33","12","Yes","No","No" +"210","3.02","98","21","11","326","90","Bad","76","11","No","Yes","No" +"211","4.36","125","41","2","357","123","Bad","47","14","No","Yes","No" +"212","9.39","117","118","14","445","120","Medium","32","15","Yes","Yes","Yes" +"213","12.04","145","69","19","501","105","Medium","45","11","Yes","Yes","Yes" +"214","8.23","149","84","5","220","139","Medium","33","10","Yes","Yes","Yes" +"215","4.83","115","115","3","48","107","Medium","73","18","Yes","Yes","No" +"216","2.34","116","83","15","170","144","Bad","71","11","Yes","Yes","No" +"217","5.73","141","33","0","243","144","Medium","34","17","Yes","No","No" +"218","4.34","106","44","0","481","111","Medium","70","14","No","No","No" +"219","9.7","138","61","12","156","120","Medium","25","14","Yes","Yes","Yes" +"220","10.62","116","79","19","359","116","Good","58","17","Yes","Yes","Yes" +"221","10.59","131","120","15","262","124","Medium","30","10","Yes","Yes","Yes" +"222","6.43","124","44","0","125","107","Medium","80","11","Yes","No","No" +"223","7.49","136","119","6","178","145","Medium","35","13","Yes","Yes","No" +"224","3.45","110","45","9","276","125","Medium","62","14","Yes","Yes","No" +"225","4.1","134","82","0","464","141","Medium","48","13","No","No","No" +"226","6.68","107","25","0","412","82","Bad","36","14","Yes","No","No" +"227","7.8","119","33","0","245","122","Good","56","14","Yes","No","No" +"228","8.69","113","64","10","68","101","Medium","57","16","Yes","Yes","Yes" +"229","5.4","149","73","13","381","163","Bad","26","11","No","Yes","No" +"230","11.19","98","104","0","404","72","Medium","27","18","No","No","Yes" +"231","5.16","115","60","0","119","114","Bad","38","14","No","No","No" +"232","8.09","132","69","0","123","122","Medium","27","11","No","No","Yes" +"233","13.14","137","80","10","24","105","Good","61","15","Yes","Yes","Yes" +"234","8.65","123","76","18","218","120","Medium","29","14","No","Yes","Yes" +"235","9.43","115","62","11","289","129","Good","56","16","No","Yes","Yes" +"236","5.53","126","32","8","95","132","Medium","50","17","Yes","Yes","No" +"237","9.32","141","34","16","361","108","Medium","69","10","Yes","Yes","Yes" +"238","9.62","151","28","8","499","135","Medium","48","10","Yes","Yes","Yes" +"239","7.36","121","24","0","200","133","Good","73","13","Yes","No","No" +"240","3.89","123","105","0","149","118","Bad","62","16","Yes","Yes","No" +"241","10.31","159","80","0","362","121","Medium","26","18","Yes","No","Yes" +"242","12.01","136","63","0","160","94","Medium","38","12","Yes","No","Yes" +"243","4.68","124","46","0","199","135","Medium","52","14","No","No","No" +"244","7.82","124","25","13","87","110","Medium","57","10","Yes","Yes","No" +"245","8.78","130","30","0","391","100","Medium","26","18","Yes","No","Yes" +"246","10","114","43","0","199","88","Good","57","10","No","Yes","Yes" +"247","6.9","120","56","20","266","90","Bad","78","18","Yes","Yes","No" +"248","5.04","123","114","0","298","151","Bad","34","16","Yes","No","No" +"249","5.36","111","52","0","12","101","Medium","61","11","Yes","Yes","No" +"250","5.05","125","67","0","86","117","Bad","65","11","Yes","No","No" +"251","9.16","137","105","10","435","156","Good","72","14","Yes","Yes","Yes" +"252","3.72","139","111","5","310","132","Bad","62","13","Yes","Yes","No" +"253","8.31","133","97","0","70","117","Medium","32","16","Yes","No","Yes" +"254","5.64","124","24","5","288","122","Medium","57","12","No","Yes","No" +"255","9.58","108","104","23","353","129","Good","37","17","Yes","Yes","Yes" +"256","7.71","123","81","8","198","81","Bad","80","15","Yes","Yes","No" +"257","4.2","147","40","0","277","144","Medium","73","10","Yes","No","No" +"258","8.67","125","62","14","477","112","Medium","80","13","Yes","Yes","Yes" +"259","3.47","108","38","0","251","81","Bad","72","14","No","No","No" +"260","5.12","123","36","10","467","100","Bad","74","11","No","Yes","No" +"261","7.67","129","117","8","400","101","Bad","36","10","Yes","Yes","No" +"262","5.71","121","42","4","188","118","Medium","54","15","Yes","Yes","No" +"263","6.37","120","77","15","86","132","Medium","48","18","Yes","Yes","No" +"264","7.77","116","26","6","434","115","Medium","25","17","Yes","Yes","No" +"265","6.95","128","29","5","324","159","Good","31","15","Yes","Yes","No" +"266","5.31","130","35","10","402","129","Bad","39","17","Yes","Yes","No" +"267","9.1","128","93","12","343","112","Good","73","17","No","Yes","Yes" +"268","5.83","134","82","7","473","112","Bad","51","12","No","Yes","No" +"269","6.53","123","57","0","66","105","Medium","39","11","Yes","No","No" +"270","5.01","159","69","0","438","166","Medium","46","17","Yes","No","No" +"271","11.99","119","26","0","284","89","Good","26","10","Yes","No","Yes" +"272","4.55","111","56","0","504","110","Medium","62","16","Yes","No","No" +"273","12.98","113","33","0","14","63","Good","38","12","Yes","No","Yes" +"274","10.04","116","106","8","244","86","Medium","58","12","Yes","Yes","Yes" +"275","7.22","135","93","2","67","119","Medium","34","11","Yes","Yes","No" +"276","6.67","107","119","11","210","132","Medium","53","11","Yes","Yes","No" +"277","6.93","135","69","14","296","130","Medium","73","15","Yes","Yes","No" +"278","7.8","136","48","12","326","125","Medium","36","16","Yes","Yes","No" +"279","7.22","114","113","2","129","151","Good","40","15","No","Yes","No" +"280","3.42","141","57","13","376","158","Medium","64","18","Yes","Yes","No" +"281","2.86","121","86","10","496","145","Bad","51","10","Yes","Yes","No" +"282","11.19","122","69","7","303","105","Good","45","16","No","Yes","Yes" +"283","7.74","150","96","0","80","154","Good","61","11","Yes","No","No" +"284","5.36","135","110","0","112","117","Medium","80","16","No","No","No" +"285","6.97","106","46","11","414","96","Bad","79","17","No","No","No" +"286","7.6","146","26","11","261","131","Medium","39","10","Yes","Yes","No" +"287","7.53","117","118","11","429","113","Medium","67","18","No","Yes","No" +"288","6.88","95","44","4","208","72","Bad","44","17","Yes","Yes","No" +"289","6.98","116","40","0","74","97","Medium","76","15","No","No","No" +"290","8.75","143","77","25","448","156","Medium","43","17","Yes","Yes","Yes" +"291","9.49","107","111","14","400","103","Medium","41","11","No","Yes","Yes" +"292","6.64","118","70","0","106","89","Bad","39","17","Yes","No","No" +"293","11.82","113","66","16","322","74","Good","76","15","Yes","Yes","Yes" +"294","11.28","123","84","0","74","89","Good","59","10","Yes","No","Yes" +"295","12.66","148","76","3","126","99","Good","60","11","Yes","Yes","Yes" +"296","4.21","118","35","14","502","137","Medium","79","10","No","Yes","No" +"297","8.21","127","44","13","160","123","Good","63","18","Yes","Yes","Yes" +"298","3.07","118","83","13","276","104","Bad","75","10","Yes","Yes","No" +"299","10.98","148","63","0","312","130","Good","63","15","Yes","No","Yes" +"300","9.4","135","40","17","497","96","Medium","54","17","No","Yes","Yes" +"301","8.57","116","78","1","158","99","Medium","45","11","Yes","Yes","Yes" +"302","7.41","99","93","0","198","87","Medium","57","16","Yes","Yes","No" +"303","5.28","108","77","13","388","110","Bad","74","14","Yes","Yes","No" +"304","10.01","133","52","16","290","99","Medium","43","11","Yes","Yes","Yes" +"305","11.93","123","98","12","408","134","Good","29","10","Yes","Yes","Yes" +"306","8.03","115","29","26","394","132","Medium","33","13","Yes","Yes","Yes" +"307","4.78","131","32","1","85","133","Medium","48","12","Yes","Yes","No" +"308","5.9","138","92","0","13","120","Bad","61","12","Yes","No","No" +"309","9.24","126","80","19","436","126","Medium","52","10","Yes","Yes","Yes" +"310","11.18","131","111","13","33","80","Bad","68","18","Yes","Yes","Yes" +"311","9.53","175","65","29","419","166","Medium","53","12","Yes","Yes","Yes" +"312","6.15","146","68","12","328","132","Bad","51","14","Yes","Yes","No" +"313","6.8","137","117","5","337","135","Bad","38","10","Yes","Yes","No" +"314","9.33","103","81","3","491","54","Medium","66","13","Yes","No","Yes" +"315","7.72","133","33","10","333","129","Good","71","14","Yes","Yes","No" +"316","6.39","131","21","8","220","171","Good","29","14","Yes","Yes","No" +"317","15.63","122","36","5","369","72","Good","35","10","Yes","Yes","Yes" +"318","6.41","142","30","0","472","136","Good","80","15","No","No","No" +"319","10.08","116","72","10","456","130","Good","41","14","No","Yes","Yes" +"320","6.97","127","45","19","459","129","Medium","57","11","No","Yes","No" +"321","5.86","136","70","12","171","152","Medium","44","18","Yes","Yes","No" +"322","7.52","123","39","5","499","98","Medium","34","15","Yes","No","No" +"323","9.16","140","50","10","300","139","Good","60","15","Yes","Yes","Yes" +"324","10.36","107","105","18","428","103","Medium","34","12","Yes","Yes","Yes" +"325","2.66","136","65","4","133","150","Bad","53","13","Yes","Yes","No" +"326","11.7","144","69","11","131","104","Medium","47","11","Yes","Yes","Yes" +"327","4.69","133","30","0","152","122","Medium","53","17","Yes","No","No" +"328","6.23","112","38","17","316","104","Medium","80","16","Yes","Yes","No" +"329","3.15","117","66","1","65","111","Bad","55","11","Yes","Yes","No" +"330","11.27","100","54","9","433","89","Good","45","12","Yes","Yes","Yes" +"331","4.99","122","59","0","501","112","Bad","32","14","No","No","No" +"332","10.1","135","63","15","213","134","Medium","32","10","Yes","Yes","Yes" +"333","5.74","106","33","20","354","104","Medium","61","12","Yes","Yes","No" +"334","5.87","136","60","7","303","147","Medium","41","10","Yes","Yes","No" +"335","7.63","93","117","9","489","83","Bad","42","13","Yes","Yes","No" +"336","6.18","120","70","15","464","110","Medium","72","15","Yes","Yes","No" +"337","5.17","138","35","6","60","143","Bad","28","18","Yes","No","No" +"338","8.61","130","38","0","283","102","Medium","80","15","Yes","No","Yes" +"339","5.97","112","24","0","164","101","Medium","45","11","Yes","No","No" +"340","11.54","134","44","4","219","126","Good","44","15","Yes","Yes","Yes" +"341","7.5","140","29","0","105","91","Bad","43","16","Yes","No","No" +"342","7.38","98","120","0","268","93","Medium","72","10","No","No","No" +"343","7.81","137","102","13","422","118","Medium","71","10","No","Yes","No" +"344","5.99","117","42","10","371","121","Bad","26","14","Yes","Yes","No" +"345","8.43","138","80","0","108","126","Good","70","13","No","Yes","Yes" +"346","4.81","121","68","0","279","149","Good","79","12","Yes","No","No" +"347","8.97","132","107","0","144","125","Medium","33","13","No","No","Yes" +"348","6.88","96","39","0","161","112","Good","27","14","No","No","No" +"349","12.57","132","102","20","459","107","Good","49","11","Yes","Yes","Yes" +"350","9.32","134","27","18","467","96","Medium","49","14","No","Yes","Yes" +"351","8.64","111","101","17","266","91","Medium","63","17","No","Yes","Yes" +"352","10.44","124","115","16","458","105","Medium","62","16","No","Yes","Yes" +"353","13.44","133","103","14","288","122","Good","61","17","Yes","Yes","Yes" +"354","9.45","107","67","12","430","92","Medium","35","12","No","Yes","Yes" +"355","5.3","133","31","1","80","145","Medium","42","18","Yes","Yes","No" +"356","7.02","130","100","0","306","146","Good","42","11","Yes","No","No" +"357","3.58","142","109","0","111","164","Good","72","12","Yes","No","No" +"358","13.36","103","73","3","276","72","Medium","34","15","Yes","Yes","Yes" +"359","4.17","123","96","10","71","118","Bad","69","11","Yes","Yes","No" +"360","3.13","130","62","11","396","130","Bad","66","14","Yes","Yes","No" +"361","8.77","118","86","7","265","114","Good","52","15","No","Yes","Yes" +"362","8.68","131","25","10","183","104","Medium","56","15","No","Yes","Yes" +"363","5.25","131","55","0","26","110","Bad","79","12","Yes","Yes","No" +"364","10.26","111","75","1","377","108","Good","25","12","Yes","No","Yes" +"365","10.5","122","21","16","488","131","Good","30","14","Yes","Yes","Yes" +"366","6.53","154","30","0","122","162","Medium","57","17","No","No","No" +"367","5.98","124","56","11","447","134","Medium","53","12","No","Yes","No" +"368","14.37","95","106","0","256","53","Good","52","17","Yes","No","Yes" +"369","10.71","109","22","10","348","79","Good","74","14","No","Yes","Yes" +"370","10.26","135","100","22","463","122","Medium","36","14","Yes","Yes","Yes" +"371","7.68","126","41","22","403","119","Bad","42","12","Yes","Yes","No" +"372","9.08","152","81","0","191","126","Medium","54","16","Yes","No","Yes" +"373","7.8","121","50","0","508","98","Medium","65","11","No","No","No" +"374","5.58","137","71","0","402","116","Medium","78","17","Yes","No","No" +"375","9.44","131","47","7","90","118","Medium","47","12","Yes","Yes","Yes" +"376","7.9","132","46","4","206","124","Medium","73","11","Yes","No","No" +"377","16.27","141","60","19","319","92","Good","44","11","Yes","Yes","Yes" +"378","6.81","132","61","0","263","125","Medium","41","12","No","No","No" +"379","6.11","133","88","3","105","119","Medium","79","12","Yes","Yes","No" +"380","5.81","125","111","0","404","107","Bad","54","15","Yes","No","No" +"381","9.64","106","64","10","17","89","Medium","68","17","Yes","Yes","Yes" +"382","3.9","124","65","21","496","151","Bad","77","13","Yes","Yes","No" +"383","4.95","121","28","19","315","121","Medium","66","14","Yes","Yes","No" +"384","9.35","98","117","0","76","68","Medium","63","10","Yes","No","Yes" +"385","12.85","123","37","15","348","112","Good","28","12","Yes","Yes","Yes" +"386","5.87","131","73","13","455","132","Medium","62","17","Yes","Yes","No" +"387","5.32","152","116","0","170","160","Medium","39","16","Yes","No","No" +"388","8.67","142","73","14","238","115","Medium","73","14","No","Yes","Yes" +"389","8.14","135","89","11","245","78","Bad","79","16","Yes","Yes","Yes" +"390","8.44","128","42","8","328","107","Medium","35","12","Yes","Yes","Yes" +"391","5.47","108","75","9","61","111","Medium","67","12","Yes","Yes","No" +"392","6.1","153","63","0","49","124","Bad","56","16","Yes","No","No" +"393","4.53","129","42","13","315","130","Bad","34","13","Yes","Yes","No" +"394","5.57","109","51","10","26","120","Medium","30","17","No","Yes","No" +"395","5.35","130","58","19","366","139","Bad","33","16","Yes","Yes","No" +"396","12.57","138","108","17","203","128","Good","33","14","Yes","Yes","Yes" +"397","6.14","139","23","3","37","120","Medium","55","11","No","Yes","No" +"398","7.41","162","26","12","368","159","Medium","40","18","Yes","Yes","No" +"399","5.94","100","79","7","284","95","Bad","50","12","Yes","Yes","No" +"400","9.71","134","37","0","27","120","Good","49","16","Yes","Yes","Yes" diff --git a/tests/data_examples/data_full_input_manifest/in/tables/sample.csv.manifest b/tests/data_examples/data_full_input_manifest/in/tables/sample.csv.manifest new file mode 100644 index 0000000..fa5f602 --- /dev/null +++ b/tests/data_examples/data_full_input_manifest/in/tables/sample.csv.manifest @@ -0,0 +1 @@ +{"id": "in.c-main.test", "uri": "https://connection.keboola.com//v2/storage/tables/in.c-main.test", "name": "sample.csv", "created": "2015-11-02T09:11:37+0100", "last_change_date": "2015-11-02T09:11:37+0100", "last_import_date": "2015-11-02T09:11:37+0100", "rows_count": 400, "data_size_bytes": 81920, "is_alias": false, "indexed_columns": ["x"], "primary_key": ["x"], "column_metadata": {"x": [{"key": "foo", "value": "gogo"}]}, "columns": ["x", "Sales", "CompPrice", "Income", "Advertising", "Population", "Price", "ShelveLoc", "Age", "Education", "Urban", "US", "High"]} \ No newline at end of file diff --git a/tests/test_interface.py b/tests/test_interface.py index 33ec4ff..75b0be1 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -685,6 +685,35 @@ def test_convert_new_to_old_manifest_storage_param(self): 'write_always': False }, old_manifest) + def test_full_input_manifest(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data_examples', 'data_full_input_manifest') + os.environ["KBC_DATADIR"] = path + + ci = CommonInterface() + tables = ci.get_input_tables_definitions() + + ci.write_manifests([tables[0]]) + manifest_filename = tables[0].full_path + '.manifest' + with open(manifest_filename) as manifest_file: + old_manifest = json.load(manifest_file) + + self.assertEqual({ + 'id': 'in.c-main.test', + 'uri': 'https://connection.keboola.com//v2/storage/tables/in.c-main.test', + 'name': 'sample.csv', + 'created': '2015-11-02T09:11:37+0100', + 'last_change_date': '2015-11-02T09:11:37+0100', + 'last_import_date': '2015-11-02T09:11:37+0100', + 'rows_count': 400, + 'data_size_bytes': 81920, + 'is_alias': False, + 'indexed_columns': ['x'], + 'primary_key': ['x'], + 'column_metadata': {'x': [{'key': 'foo', 'value': 'gogo'}]}, + 'columns': ['x', 'Sales', 'CompPrice', 'Income', 'Advertising', 'Population', 'Price', 'ShelveLoc', 'Age', + 'Education', 'Urban', 'US', 'High'] + }, old_manifest) + class TestConfiguration(unittest.TestCase):