ci: Lint unused imports

- requirements-dev: Added autoflake
- ci: Added unused import check
- ci: run: Added lint imports call in run_imports
- CHANGELOG: Updated
- service: dev: Updated RemoveUnusedImports to fix branch name fetching error
- service: dev: Removed unused code from RemoveUnusedImports
- source: dfpreprocess: Removed unused import
- operations: data: Removed unused imports
- tests: service: dev: Added test for RemoveUnusedImports CMD

Fixes: #1190

.ci/run.sh

@@ -196,6 +196,19 @@ function run_commit(){
   fi
 }
 
+function run_imports(){
+  dffml service dev lint imports
+  if [[ -z $(git status -s) ]]
+  then
+    echo "Yay ! No unused imports found"
+  else
+    echo "There maybe unused imports in the following files:"
+    git status -s  | grep "M" | awk '{print $2}'
+    exit 1
+  fi
+
+}
+
 function run_docs() {
   export GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
 
@@ -317,6 +330,8 @@ elif [ "x${1}" == "xstyle" ]; then
   run_style
 elif [ "x${1}" == "xcommit" ]; then
   run_commit
+elif [ "x${1}" == "ximport" ]; then
+  run_imports
 elif [ "x${1}" == "xdocs" ]; then
   run_docs
 elif [ "x${1}" == "xlines" ]; then

.github/workflows/testing.yml

@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       max-parallel: 40
       matrix:
-        check: [changelog, whitespace, style, commit, docs, lines]
+        check: [changelog, whitespace, style, commit, import, docs, lines]
         python-version: [3.7]
         node-version: [12.x]
 

CHANGELOG.md

@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Operations for data cleanup
 - Examples of using data cleanup operations
   https://intel.github.io/dffml/examples/data_cleanup/index.html
+- Dev CMD to remove unused imports, `$ dffml service dev lint imports`
 ### Changed
 - Calls to hashlib now go through helper functions
 - Build docs using `dffml service dev docs`

dffml/service/dev.py

@@ -58,6 +58,23 @@
 REPO_ROOT = pathlib.Path(__file__).parents[2]
 
 
+async def get_cmd_output(cmd: List[str]):
+    print(f"$ {' '.join(cmd)}")
+
+    proc = await asyncio.create_subprocess_shell(
+        " ".join(cmd),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        cwd=REPO_ROOT,
+    )
+    await proc.wait()
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(stderr)
+    output = stdout.decode().strip()
+    return output
+
+
 def create_from_skel(plugin_type):
     """
     Copies samples out of skel/ and does re-naming.
@@ -750,6 +767,25 @@ async def run(self):
             self.logger.debug("Updated version file %s", version_file)
 
 
+class RemoveUnusedImports(CMD):
+    async def _run_autoflake(self):
+        cmd = [
+            "git",
+            "ls-files",
+            "'*.py'",
+            "|",
+            "xargs",
+            "autoflake",
+            "--in-place",
+            "--remove-all-unused-imports",
+            "--ignore-init-module-imports",
+        ]
+        await get_cmd_output(cmd)
+
+    async def run(self):
+        await self._run_autoflake()
+
+
 class CommitLintError(Exception):
     pass
 
@@ -823,34 +859,18 @@ async def _get_file_mutations(self):
         }
         return mutations
 
-    async def _get_cmd_output(self, cmd: List[str]):
-        print(f"$ {' '.join(cmd)}")
-
-        proc = await asyncio.create_subprocess_shell(
-            " ".join(cmd),
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            cwd=REPO_ROOT,
-        )
-        await proc.wait()
-        stdout, stderr = await proc.communicate()
-        if proc.returncode != 0:
-            raise RuntimeError(stderr)
-        output = stdout.decode().strip()
-        return output
-
     async def _get_relevant_commits(self):
         #! This needs to change when master is renamed to main.
         cmd = ["git", "cherry", "-v", "origin/master"]
-        commits = await self._get_cmd_output(cmd)
+        commits = await get_cmd_output(cmd)
         commits_list = [
             " ".join(line.split()[2:]) for line in commits.split("\n")
         ]
         return commits_list
 
     async def _get_commmit_details(self, msg):
         cmd = ["git", "log", f"""--grep='{msg}'"""]
-        commit_details = await self._get_cmd_output(cmd)
+        commit_details = await get_cmd_output(cmd)
         return commit_details
 
     async def _get_all_exts(self,):
@@ -861,7 +881,7 @@ async def _get_all_exts(self,):
             "HEAD",
             "--name-only",
         ]
-        tracked_files = await self._get_cmd_output(cmd)
+        tracked_files = await get_cmd_output(cmd)
         tracked_files = tracked_files.split("\n")
         extentions = set()
         for file in tracked_files:
@@ -927,6 +947,7 @@ class Lint(CMD):
     """
 
     commits = LintCommits
+    imports = RemoveUnusedImports
 
 
 class Bump(CMD):

dffml/source/dfpreprocess.py

@@ -3,7 +3,7 @@
 
 from ..base import config, field
 from ..configloader.configloader import BaseConfigLoader
-from ..df.types import Definition, DataFlow, Input
+from ..df.types import DataFlow, Input
 from ..df.base import BaseOrchestrator
 from ..feature import Features
 from ..record import Record

operations/data/dffml_operations_data/operations.py

@@ -1,8 +1,6 @@
-from typing import List
-
 import numpy as np
 from sklearn.decomposition import PCA, TruncatedSVD
-from sklearn.preprocessing import OneHotEncoder, StandardScaler, OrdinalEncoder
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
 from sklearn.impute import SimpleImputer
 
 from dffml.df.base import op

requirements-dev.txt

@@ -10,6 +10,7 @@ nbsphinx
 nbsphinx_link
 black==19.10b0
 jsbeautifier>=1.14.0
+autoflake
 twine
 # Test requirements
 httptest>=0.0.15

tests/service/test_dev.py

@@ -517,3 +517,25 @@ async def test_shouldnot_validate(self):
                     ]
                 )
             )
+
+
+class TestRemoveUnusedImports(AsyncTestCase):
+    async def test_cmd_execution(self):
+        if not is_develop("dffml"):
+            self.skipTest("dffml not installed in development mode")
+
+        stdout = io.StringIO()
+
+        with unittest.mock.patch(
+            "asyncio.create_subprocess_exec", new=mkexec()
+        ), contextlib.redirect_stdout(stdout):
+            await Develop.cli("lint", "imports")
+
+        self.assertEqual(
+            stdout.getvalue().strip(),
+            inspect.cleandoc(
+                """
+                $ git ls-files '*.py' | xargs autoflake --in-place --remove-all-unused-imports --ignore-init-module-imports
+                """
+            ),
+        )