Cleaning non utf8 strings

package.json

@@ -1,6 +1,6 @@
 {
   "name": "deep-cuts",
-  "version": "2.2.0",
+  "version": "2.3.1",
   "description": "Useful utilities and rare b-sides.",
   "author": "Trevor Ewen",
   "license": "MIT",
@@ -59,9 +59,10 @@
     "qs": "^6.5.2"
   },
   "devDependencies": {
-    "@size-limit/preset-small-lib": "^5.0.1",
+    "@size-limit/preset-small-lib": "^7.0.4",
     "@types/jest": "^25.2.3",
     "@types/qs": "^6.9.7",
+    "axios": "^0.24.0",
     "data-mining-tools": "^1.1.1",
     "husky": "^7.0.1",
     "size-limit": "^5.0.1",

src/__tests__/stream.test.ts

@@ -48,6 +48,7 @@ describe('stream', () => {
           fs.createReadStream(path.join(__dirname, 'fixtures/bad.json'))
         );
       } catch (e) {
+        // @ts-ignore
         expect(e.message).toEqual(
           'Stream did not resolve to a JSON object, you may need to process it another way.'
         );

src/__tests__/string.test.ts

@@ -1,4 +1,5 @@
 import { stringToBoolean, escapeForRegExp } from '../';
+import { cleanSpecialCharacters } from '../string';
 
 describe('string', () => {
   describe('stringToBoolean()', () => {
@@ -146,4 +147,78 @@ describe('string', () => {
       );
     });
   });
+
+  describe('cleanSpecialCharacters()', () => {
+    it('should return an empty string if passed undefined', () => {
+      // @ts-ignore
+      expect(cleanSpecialCharacters(undefined)).toEqual('');
+    });
+
+    it('should return an empty string if passed null', () => {
+      // @ts-ignore
+      expect(cleanSpecialCharacters(null)).toEqual('');
+    });
+
+    it('should return an empty string if passed an empty string', () => {
+      expect(cleanSpecialCharacters('')).toEqual('');
+    });
+
+    it('should return a string with no special characters as-is', () => {
+      expect(cleanSpecialCharacters('Koolaid, Punch, Results.xlsx')).toEqual(
+        'Koolaid, Punch, Results.xlsx'
+      );
+    });
+
+    it('should return a string with no special characters as-is, even if all spaces', () => {
+      expect(cleanSpecialCharacters('    ')).toEqual('    ');
+    });
+
+    it('should play nice with numbers', () => {
+      // @ts-ignore
+      expect(cleanSpecialCharacters(25.624)).toEqual('25.624');
+    });
+
+    it('should play nice with numbers in a string', () => {
+      expect(cleanSpecialCharacters('56 Signs.pdf')).toEqual('56 Signs.pdf');
+    });
+
+    it('should play nice with punctuation', () => {
+      expect(
+        cleanSpecialCharacters('Reginald. Sr. Officer of the war $5.00')
+      ).toEqual('Reginald. Sr. Officer of the war $5.00');
+    });
+
+    it('should remove all characters with unicode code above 127', () => {
+      expect(
+        cleanSpecialCharacters(
+          `Hello, ${String.fromCharCode(890)} my name is ${String.fromCharCode(
+            129
+          )}hill`
+        )
+      ).toEqual('Hello,  my name is hill');
+    });
+
+    it('should be able to specify a replacement for unicode characters above 127', () => {
+      expect(
+        cleanSpecialCharacters(
+          `Hello, ${String.fromCharCode(1200)} my name is ${String.fromCharCode(
+            135
+          )}hill`,
+          'X'
+        )
+      ).toEqual('Hello, X my name is Xhill');
+    });
+
+    it('should play nice with characters right at the edge', () => {
+      expect(
+        cleanSpecialCharacters(
+          [
+            String.fromCharCode(50),
+            String.fromCharCode(60),
+            String.fromCharCode(126),
+          ].join('')
+        )
+      ).toEqual('2<~');
+    });
+  });
 });

src/array.ts

@@ -4,14 +4,9 @@ export async function tailRecursion<T>(
   payload: any[] = []
 ): Promise<any[]> {
   if (collection?.length) {
-    const first: T = collection[0];
-    if (first) {
-      const response: any = await operation(first);
-      return tailRecursion(
-        collection.slice(1),
-        operation,
-        payload.concat(response)
-      );
+    for (const item of collection) {
+      const response: any = await operation(item);
+      payload.push(response);
     }
   }
   return payload;

src/string.ts

@@ -16,3 +16,18 @@ export function escapeForRegExp(str: string): string {
     : str;
   /* eslint-enable no-useless-escape */
 }
+
+export function cleanSpecialCharacters(
+  str: string,
+  replacement: string = ''
+): string {
+  return String(str || '')
+    .split('')
+    .map(char => {
+      if (char.charCodeAt(0) > 127) {
+        return replacement;
+      }
+      return char;
+    })
+    .join('');
+}