Merge branch 'development'

CHANGES.rst

@@ -1,8 +1,16 @@
+**2021-11-01**
+
+*Version 0.3.0*
+
+- Process and transform row values using field processors `read more <https://csvio.readthedocs.io/en/latest/csvio.fieldprocessor.html>`_
+- Update/Fix documentation
+- Refactor/Add tests
+
 **2021-10-03**
 
 *Version 0.2.0*
 
-- Construct a nested dictionary from rows based on a list of ordered column names `... read more <https://csvio.readthedocs.io/en/latest/csvio.csvbase.html#csvio.csvbase.CSVBase.rows_to_nested_dicts>`_
+- Construct a nested dictionary from rows based on a list of ordered column names `read more <https://csvio.readthedocs.io/en/latest/csvio.csvbase.html#csvio.csvbase.CSVBase.rows_to_nested_dicts>`_
 - Update/Fix documentation
 - Refactor/Add tests
 

README.rst

@@ -75,7 +75,7 @@ Reading CSVs
       }
    ]
 
-CSV file contents:
+*CSV file contents:*
 
 .. code-block:: bash
 
@@ -125,3 +125,5 @@ written with the following contents.
       Small Strawberries,Strawberry,4
 
 .. ignore-below-marker
+
+`Readthedocs for more example code <https://csvio.readthedocs.io>`_

csvio/__init__.py

@@ -21,6 +21,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 from .csvreader import CSVReader
 from .csvwriter import CSVWriter

csvio/csvbase.py

@@ -24,6 +24,7 @@
 from typing import Any, Dict, List
 
 from .filebase import FileBase
+from .utils.types import FN, KW, RS
 
 
 class CSVBase(FileBase):
@@ -48,47 +49,47 @@ class CSVBase(FileBase):
     def __init__(
         self,
         filename: str,
-        open_kwargs: Dict[str, Any] = {},
-        csv_kwargs: Dict[str, Any] = {},
+        open_kwargs: KW = {},
+        csv_kwargs: KW = {},
     ) -> None:
 
         super().__init__(filename)
 
         self._open_kwargs = open_kwargs
         self._csv_kwargs = csv_kwargs
 
-        self._fieldnames: List[str] = []
-        self._rows: List[Dict[str, Any]] = []
+        self._fieldnames: FN = []
+        self._rows: RS = []
 
     @property
-    def open_kwargs(self) -> Dict[str, Any]:
+    def open_kwargs(self) -> KW:
         """
         :return: A dictionary of key, value pairs that should be passed to the
             open method within this class.
         """
         return self._open_kwargs
 
     @property
-    def csv_kwargs(self) -> Dict[str, Any]:
+    def csv_kwargs(self) -> KW:
         """
         :return: A dictionary of key, value pairs that should be passed to the
             DictReader constructor within this class.
         """
         return self._csv_kwargs
 
     @property
-    def fieldnames(self) -> List[str]:
+    def fieldnames(self) -> FN:
         """
         :return: List of column headings
         """
         return list(self._fieldnames)
 
     @fieldnames.setter
-    def fieldnames(self, fieldnames: List[str]) -> None:
+    def fieldnames(self, fieldnames: FN) -> None:
         self._fieldnames = fieldnames
 
     @property
-    def rows(self) -> List[Dict[str, Any]]:
+    def rows(self) -> RS:
         """
         :return: A list of dictionaries where each item in it represents a row
             in the CSV file. Each dictionary in the list maps the column
@@ -97,7 +98,7 @@ def rows(self) -> List[Dict[str, Any]]:
         return self._rows
 
     @rows.setter
-    def rows(self, rows: List[Dict[str, Any]]) -> None:
+    def rows(self, rows: RS) -> None:
         self._rows = rows
 
     @property
@@ -109,7 +110,7 @@ def num_rows(self) -> int:
         return len(self.rows)
 
     def _init_kwargs_dict(
-        self, dict_to_update: Dict[str, Any], args_dict: Dict[str, Any]
+        self, dict_to_update: Dict[str, Any], args_dict: KW
     ) -> None:
 
         for arg, value in args_dict.items():
@@ -118,8 +119,8 @@ def _init_kwargs_dict(
                 dict_to_update[arg] = value
 
     def rows_from_column_key(
-        self, column_name: str, rows: List[Dict[str, Any]] = None
-    ) -> Dict[str, List[Dict[str, Any]]]:
+        self, column_name: str, rows: RS = None
+    ) -> Dict[str, RS]:
         """
         Collect all the rows in the ``rows`` parameter that have the same
         values for the column defined in the ``column_name`` parameter, and
@@ -141,7 +142,7 @@ def rows_from_column_key(
         :return: A dictionary constructed using the logic as explained above.
         """
 
-        ret_dict: Dict[str, Any] = {}
+        ret_dict: Dict[str, RS] = {}
         rows = rows or self.rows
 
         for row in rows:
@@ -150,7 +151,7 @@ def rows_from_column_key(
         return ret_dict
 
     def rows_to_nested_dicts(
-        self, column_order: List[str], rows: List[Dict[str, Any]] = None
+        self, column_order: List[str], rows: RS = None
     ) -> Dict[str, Any]:
         """
         Collect all values of columns that are the same and construct a nested

csvio/csvreader.py

@@ -23,9 +23,10 @@
 # SOFTWARE.
 import csv
 import traceback
-from typing import Any, Dict, List
 
 from .csvbase import CSVBase
+from .processors import FieldProcessor
+from .utils.types import FN, KW, RS, R
 
 
 class CSVReader(CSVBase):
@@ -45,6 +46,14 @@ class CSVReader(CSVBase):
         *filename* argument of this Class's constructor.
     :type fieldnames: optional
 
+    :param fieldprocessor:
+        An instance of the
+        :py:class:`~csvio.processors.field_processor.FieldProcessor`
+        object. The processor functions defined in the
+        :py:class:`~csvio.processors.field_processor.FieldProcessor`
+        object are applied to the rows in the CSV after they read.
+    :type fieldprocessor: optional
+
     :param open_kwargs:
         A dictionary of key, value pairs that should be passed to the open
         method within this class.
@@ -55,71 +64,40 @@ class CSVReader(CSVBase):
         DictReader constructor within this class.
     :type csv_kwargs: optional
 
-    Usage:
-
-    .. doctest::
-
-        >>> from csvio import CSVReader
-        >>> reader = CSVReader("fruit_stock.csv")
-        >>> reader.fieldnames
-        ['Supplier', 'Fruit', 'Quantity']
-
-        >>> len(reader.rows)
-        4
-
-        >>> import json
-        >>> print(json.dumps(reader.rows, indent=4))
-        [
-            {
-                "Supplier": "Big Apple",
-                "Fruit": "Apple",
-                "Quantity": "1"
-            },
-            {
-                "Supplier": "Big Melons",
-                "Fruit": "Melons",
-                "Quantity": "2"
-            },
-            {
-                "Supplier": "Big Mangoes",
-                "Fruit": "Mango",
-                "Quantity": "3"
-            },
-            {
-                "Supplier": "Small Strawberries",
-                "Fruit": "Strawberry",
-                "Quantity": "4"
-            }
-        ]
-
-    CSV file contents:
-
-    .. code-block:: bash
-
-            Supplier,Fruit,Quantity
-            Big Apple,Apple,1
-            Big Melons,Melons,2
-            Long Mangoes,Mango,3
-            Small Strawberries,Strawberry,4
+    **CSVReader usage without** ``fieldprocessor``:
+
+    .. include:: examples/csvio.csvreader.rst
+        :start-after: start-csvreader
+        :end-before: end-csvreader
+
+    .. _csvreader_fp_usage:
+
+    **CSVReader usage with** ``fieldprocessor``
+
+    .. include:: examples/csvio.fieldprocessor.rst
+        :start-after: start-csvreader_field_processor
+        :end-before: end-csvreader_field_processor
 
     """
 
     def __init__(
         self,
         filename: str,
-        fieldnames: List[str] = [],
-        open_kwargs: Dict[str, str] = {},
-        csv_kwargs: Dict[str, Any] = {},
+        fieldprocessor: FieldProcessor = None,
+        fieldnames: FN = [],
+        open_kwargs: KW = {},
+        csv_kwargs: KW = {},
     ) -> None:
 
         super().__init__(filename, open_kwargs, csv_kwargs)
 
+        self.field_processor = fieldprocessor
         self.fieldnames = fieldnames or self.__get_fieldnames()
         self.rows = self.__get_rows()
 
-    def __get_fieldnames(self) -> List[str]:
+    def __get_fieldnames(self) -> FN:
 
-        fieldnames: List[str] = []
+        fieldnames: FN = []
 
         try:
             with open(self.filepath, "r", **self.open_kwargs) as fh:
@@ -132,9 +110,9 @@ def __get_fieldnames(self) -> List[str]:
 
         return fieldnames
 
-    def __get_rows(self) -> List[Dict[str, Any]]:
+    def __get_rows(self) -> RS:
 
-        rows: List[Dict[str, Any]] = []
+        rows: RS = []
 
         try:
             with open(self.filepath, "r", **self.open_kwargs) as fh:
@@ -146,12 +124,15 @@ def __get_rows(self) -> List[Dict[str, Any]]:
 
                 for row in csv_reader:
 
-                    row_dict: Dict[str, Any] = {}
+                    row_dict: R = {}
 
                     for fieldname in self.fieldnames:
                         row_dict[fieldname] = row[fieldname]
 
-                    rows.append(row_dict)
+                    if self.field_processor is not None:
+                        rows.append(self.field_processor.process_row(row_dict))
+                    else:
+                        rows.append(row_dict)
 
         except csv.Error: