tools: overhaul csv handling for mfg_gen and nvs_partition_gen

This fixes the issue where multiline strings and strings with delimiters inside the nvs input csv file were incorrectly parsed, and adds back the ability to add comment lines anywhere in the CSV file.

The issue stems from the move away from the python built in csv module to manual parsing, which was made after moving away from using the csv module to parse mfg data.

This reverts back to using the csv module for parsing and writing csv data in both mfg_gen and nvs_partition_gen, fixes the original issue in mfg_gen and improves code quality which makes the code more readable and maintainable.

Closes #7175

components/nvs_flash/nvs_partition_generator/README.rst

@@ -43,7 +43,7 @@ Each line of a .csv file should contain 4 parameters, separated by a comma. The
 |     |           |                                                                      | Any values in these cells are ignored.              |
 +-----+-----------+----------------------------------------------------------------------+-----------------------------------------------------+
 
-.. note:: The first line of the CSV file should be the column header and it is not configurable. Comments (if provided) are allowed only as the first line of the CSV file, the following line then should always be the column header. Comments should always start with the `#` symbol.
+.. note:: The first line of the CSV file should always be the column header and it is not configurable.
 
 Below is an example dump of such a CSV file::
 
@@ -308,5 +308,5 @@ Caveats
 -------
 -  Utility does not check for duplicate keys and will write data pertaining to both keys. You need to make sure that the keys are distinct.
 -  Once a new page is created, no data will be written in the space left on the previous page. Fields in the CSV file need to be ordered in such a way as to optimize memory.
--  Utility supports using multiline strings with ``file`` type and singleline strings with ``data`` type in the CSV file.
 -  64-bit datatype is not yet supported.
+

components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py

@@ -3,19 +3,8 @@
 # esp-idf NVS partition generation tool. Tool helps in generating NVS-compatible
 # partition binary, with key-value pair entries provided via a CSV file.
 #
-# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
 #
 
 from __future__ import division, print_function
@@ -24,6 +13,7 @@
 import array
 import binascii
 import codecs
+import csv
 import datetime
 import distutils.dir_util
 import os
@@ -34,8 +24,6 @@
 from builtins import bytes, int, range
 from io import open
 
-from future.moves.itertools import zip_longest
-
 try:
     from cryptography.hazmat.backends import default_backend
     from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
@@ -547,23 +535,22 @@ def write_namespace(self, key):
     We don't have to guard re-invocation with try-except since no entry can span multiple pages.
     """
     def write_entry(self, key, value, encoding):
+        # Encoding-specific handling
         if encoding == 'hex2bin':
             value = value.strip()
             if len(value) % 2 != 0:
                 raise InputError('%s: Invalid data length. Should be multiple of 2.' % key)
             value = binascii.a2b_hex(value)
-
-        if encoding == 'base64':
+        elif encoding == 'base64':
             value = binascii.a2b_base64(value)
-
-        if encoding == 'string':
+        elif encoding == 'string':
             if type(value) == bytes:
                 value = value.decode()
             value += '\0'
 
         encoding = encoding.lower()
-        varlen_encodings = ['string', 'binary', 'hex2bin', 'base64']
-        primitive_encodings = ['u8', 'i8', 'u16', 'i16', 'u32', 'i32', 'u64', 'i64']
+        varlen_encodings = {'string', 'binary', 'hex2bin', 'base64'}
+        primitive_encodings = {'u8', 'i8', 'u16', 'i16', 'u32', 'i32', 'u64', 'i64'}
 
         if encoding in varlen_encodings:
             try:
@@ -904,46 +891,23 @@ def generate(args, is_encr_enabled=False, encr_key=None):
     if is_encr_enabled and not encr_key:
         encr_key = generate_key(args)
 
-    input_file = open(args.input, 'rt', encoding='utf8')
-    output_file = open(args.output, 'wb')
-
     with open(args.input, 'rt', encoding='utf8') as input_file,\
             open(args.output, 'wb') as output_file,\
             nvs_open(output_file, input_size, args.version, is_encrypt=is_encr_enabled, key=encr_key) as nvs_obj:
-
+        # Comments are skipped
+        reader = csv.DictReader(filter(lambda row: row[0] != '#',input_file), delimiter=',')
         if nvs_obj.version == Page.VERSION1:
             version_set = VERSION1_PRINT
         else:
             version_set = VERSION2_PRINT
-
         print('\nCreating NVS binary with version:', version_set)
 
-        line = input_file.readline().strip()
-
-        # Comments are skipped
-        while line.startswith('#'):
-            line = input_file.readline().strip()
-        if not isinstance(line, str):
-            line = line.encode('utf-8')
-
-        header = line.split(',')
-
-        while True:
-            line = input_file.readline().strip()
-            if not isinstance(line, str):
-                line = line.encode('utf-8')
-
-            value = line.split(',')
-            if len(value) == 1 and '' in value:
-                break
-
-            data = dict(zip_longest(header, value))
-
+        for row in reader:
             try:
-                # Check key length
-                if len(data['key']) > 15:
-                    raise InputError('Length of key `{}` should be <= 15 characters.'.format(data['key']))
-                write_entry(nvs_obj, data['key'], data['type'], data['encoding'], data['value'])
+                max_key_len = 15
+                if len(row['key']) > max_key_len:
+                    raise InputError('Length of key `%s` should be <= 15 characters.' % row['key'])
+                write_entry(nvs_obj, row['key'], row['type'], row['encoding'], row['value'])
             except InputError as e:
                 print(e)
                 filedir, filename = os.path.split(args.output)

components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv

@@ -6,7 +6,11 @@ dummyI8Key,data,i8,-128
 dummyU16Key,data,u16,32768
 dummyU32Key,data,u32,4294967295
 dummyI32Key,data,i32,-2147483648
-dummyStringKey,data,string,0A:0B:0C:0D:0E:0F
+dummyStringKey,data,string,"Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Fusce quis risus justo.
+Suspendisse egestas in nisi sit amet auctor.
+Pellentesque rhoncus dictum sodales.
+In justo erat, viverra at interdum eget, interdum vel dui."
 dummyHex2BinKey,data,hex2bin,010203abcdef
 dummyBase64Key,data,base64,MTIzYWJj
 hexFileKey,file,hex2bin,testdata/sample.hex

components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv

@@ -6,7 +6,11 @@ dummyI8Key,data,i8,-128
 dummyU16Key,data,u16,32768
 dummyU32Key,data,u32,4294967295
 dummyI32Key,data,i32,-2147483648
-dummyStringKey,data,string,0A:0B:0C:0D:0E:0F
+dummyStringKey,data,string,"Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Fusce quis risus justo.
+Suspendisse egestas in nisi sit amet auctor.
+Pellentesque rhoncus dictum sodales.
+In justo erat, viverra at interdum eget, interdum vel dui."
 dummyHex2BinKey,data,hex2bin,010203abcdef
 dummyBase64Key,data,base64,MTIzYWJj
 hexFileKey,file,hex2bin,testdata/sample.hex

components/nvs_flash/src/intrusive_list.h

@@ -1,21 +1,14 @@
-// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 #ifndef intrusive_list_h
 #define intrusive_list_h
 
 #include <cassert>
 #include <unordered_map>
+#include <cstddef>
 
 template <typename T>
 class intrusive_list;

components/nvs_flash/src/nvs_encrypted_partition.cpp

@@ -1,16 +1,8 @@
-// Copyright 2019 Espressif Systems (Shanghai) PTE LTD
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//         http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
 
 #include <cstring>
 #include "nvs_encrypted_partition.hpp"
@@ -105,15 +97,15 @@ esp_err_t NVSEncryptedPartition::write(size_t addr, const void* src, size_t size
                                   data_unit,
                                   buf + offset,
                                   buf + offset) != 0)  {
-            delete buf;
+            delete [] buf;
             return ESP_ERR_NVS_XTS_ENCR_FAILED;
         }
     }
 
     // write data
     esp_err_t result = esp_partition_write(mESPPartition, addr, buf, size);
 
-    delete buf;
+    delete [] buf;
 
     return result;
 }

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -2682,6 +2682,96 @@ static void check_nvs_part_gen_args(SpiFlashEmulator *spi_flash_emulator,
     TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v));
     CHECK(i32v == -2147483648);
 
+    char string_buf[256];
+    const char test_str[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n"
+                             "Fusce quis risus justo.\n"
+                             "Suspendisse egestas in nisi sit amet auctor.\n"
+                             "Pellentesque rhoncus dictum sodales.\n"
+                             "In justo erat, viverra at interdum eget, interdum vel dui.";
+    size_t str_len = sizeof(test_str);
+    TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", string_buf, &str_len));
+    CHECK(strncmp(string_buf, test_str, str_len) == 0);
+
+    char buf[64] = {0};
+    uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef};
+    size_t buflen = 64;
+    int j;
+    TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen));
+    CHECK(memcmp(buf, hexdata, buflen) == 0);
+
+    uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'};
+    TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen));
+    CHECK(memcmp(buf, base64data, buflen) == 0);
+
+    buflen = 64;
+    uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef};
+    TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen));
+    CHECK(memcmp(buf, hexfiledata, buflen) == 0);
+
+    buflen = 64;
+    uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0";
+    TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen));
+    CHECK(memcmp(buf, strfiledata, buflen) == 0);
+
+    char bin_data[5200];
+    size_t bin_len = sizeof(bin_data);
+    char binfiledata[5200];
+    ifstream file;
+    file.open(filename);
+    file.read(binfiledata,5200);
+    TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
+    CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
+
+    file.close();
+
+    nvs_close(handle);
+
+    TEST_ESP_OK(nvs_flash_deinit_partition(part_name));
+}
+
+static void check_nvs_part_gen_args_mfg(SpiFlashEmulator *spi_flash_emulator,
+        char const *part_name,
+        int size,
+        char const *filename,
+        bool is_encr,
+        nvs_sec_cfg_t* xts_cfg)
+{
+    nvs_handle_t handle;
+
+    esp_partition_t esp_part;
+    esp_part.encrypted = false; // we're not testing generic flash encryption here, only the legacy NVS encryption
+    esp_part.address = 0;
+    esp_part.size = size * SPI_FLASH_SEC_SIZE;
+    strncpy(esp_part.label, part_name, PART_NAME_MAX_SIZE);
+    shared_ptr<Partition> part;
+
+    if (is_encr) {
+        NVSEncryptedPartition *enc_part = new NVSEncryptedPartition(&esp_part);
+        TEST_ESP_OK(enc_part->init(xts_cfg));
+        part.reset(enc_part);
+    } else {
+        part.reset(new PartitionEmulation(spi_flash_emulator, 0, size, part_name));
+    }
+
+    TEST_ESP_OK( NVSPartitionManager::get_instance()->init_custom(part.get(), 0, size) );
+
+    TEST_ESP_OK( nvs_open_from_partition(part_name, "dummyNamespace", NVS_READONLY, &handle));
+    uint8_t u8v;
+    TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
+    CHECK(u8v == 127);
+    int8_t i8v;
+    TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
+    CHECK(i8v == -128);
+    uint16_t u16v;
+    TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v));
+    CHECK(u16v == 32768);
+    uint32_t u32v;
+    TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v));
+    CHECK(u32v == 4294967295);
+    int32_t i32v;
+    TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v));
+    CHECK(i32v == -2147483648);
+
     char buf[64] = {0};
     size_t buflen = 64;
     TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen));
@@ -2723,7 +2813,6 @@ static void check_nvs_part_gen_args(SpiFlashEmulator *spi_flash_emulator,
     TEST_ESP_OK(nvs_flash_deinit_partition(part_name));
 }
 
-
 TEST_CASE("check and read data from partition generated via partition generation utility with multipage blob support disabled", "[nvs_part_gen]")
 {
     int status;
@@ -2886,10 +2975,10 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
     }
 
     SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
-    check_nvs_part_gen_args(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
+    check_nvs_part_gen_args_mfg(&emu1, "test", 3, "mfg_testdata/sample_singlepage_blob.bin", false, NULL);
 
     SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
-    check_nvs_part_gen_args(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
+    check_nvs_part_gen_args_mfg(&emu2, "test", 3, "testdata/sample_singlepage_blob.bin", false, NULL);
 
 
     childpid = fork();
@@ -2967,10 +3056,10 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
     }
 
     SpiFlashEmulator emu1("../../../tools/mass_mfg/host_test/bin/Test-1.bin");
-    check_nvs_part_gen_args(&emu1, "test", 4, "mfg_testdata/sample_multipage_blob.bin", false, NULL);
+    check_nvs_part_gen_args_mfg(&emu1, "test", 4, "mfg_testdata/sample_multipage_blob.bin", false, NULL);
 
     SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition.bin");
-    check_nvs_part_gen_args(&emu2, "test", 4, "testdata/sample_multipage_blob.bin", false, NULL);
+    check_nvs_part_gen_args_mfg(&emu2, "test", 4, "testdata/sample_multipage_blob.bin", false, NULL);
 
     childpid = fork();
     if (childpid == 0) {
@@ -3465,11 +3554,11 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
         cfg.tky[count] = 0x22;
     }
 
-    check_nvs_part_gen_args(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
+    check_nvs_part_gen_args_mfg(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
 
     SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin");
 
-    check_nvs_part_gen_args(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
+    check_nvs_part_gen_args_mfg(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
 
 
     childpid = fork();
@@ -3585,11 +3674,11 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
         cfg.tky[count] = buffer[count+32] & 255;
     }
 
-    check_nvs_part_gen_args(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
+    check_nvs_part_gen_args_mfg(&emu1, NVS_DEFAULT_PART_NAME, 4, "mfg_testdata/sample_multipage_blob.bin", true, &cfg);
 
     SpiFlashEmulator emu2("../nvs_partition_generator/Test-1-partition-encrypted.bin");
 
-    check_nvs_part_gen_args(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
+    check_nvs_part_gen_args_mfg(&emu2, NVS_DEFAULT_PART_NAME, 4, "testdata/sample_multipage_blob.bin", true, &cfg);
 
     childpid = fork();
     if (childpid == 0) {