more data factory fixes

IBM · Apr 26, 2024 · e66d7b8 · e66d7b8
1 parent 1c7a0ff
commit e66d7b8
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 15 deletions.
diff --git a/transforms/code/code_quality/src/code_quality_transform.py b/transforms/code/code_quality/src/code_quality_transform.py
@@ -12,10 +12,8 @@
 
 import argparse
 import os
-import sys
 
 import numpy as np
-import pyarrow
 import pyarrow as pa
 from bs4 import BeautifulSoup
 from data_processing.ray import DefaultTableTransformConfiguration, TransformLauncher

diff --git a/transforms/code/proglang_select/src/proglang_select_transform.py b/transforms/code/proglang_select/src/proglang_select_transform.py
@@ -15,7 +15,7 @@
 
 import pyarrow as pa
 import ray
-from data_processing.data_access import DataAccess, DataAccessFactory, DataAccessLocal
+from data_processing.data_access import DataAccess, DataAccessFactoryBase, DataAccessFactory
 from data_processing.ray import (
     DefaultTableTransformConfiguration,
     DefaultTableTransformRuntime,
@@ -120,7 +120,7 @@ def __init__(self, params: dict[str, Any]):
 
     def get_transform_config(
         self,
-        data_access_factory: DataAccessFactory,
+        data_access_factory: DataAccessFactoryBase,
         statistics: ActorHandle,
         files: list[str],
     ) -> dict[str, Any]:

diff --git a/transforms/universal/doc_id/src/doc_id_transform.py b/transforms/universal/doc_id/src/doc_id_transform.py
@@ -15,7 +15,7 @@
 
 import pyarrow as pa
 import ray
-from data_processing.data_access import DataAccessFactory
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.ray import (
     DefaultTableTransformConfiguration,
     DefaultTableTransformRuntime,
@@ -128,7 +128,7 @@ def __init__(self, params: dict[str, Any]):
         super().__init__(params)
 
     def get_transform_config(
-        self, data_access_factory: DataAccessFactory, statistics: ActorHandle, files: list[str]
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
     ) -> dict[str, Any]:
         """
         Set environment for filter execution

diff --git a/transforms/universal/ededup/src/ededup_transform.py b/transforms/universal/ededup/src/ededup_transform.py
@@ -15,7 +15,7 @@
 
 import pyarrow as pa
 import ray
-from data_processing.data_access import DataAccessFactory
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.ray import (
     DefaultTableTransformConfiguration,
     DefaultTableTransformRuntime,
@@ -178,7 +178,7 @@ def __init__(self, params: dict[str, Any]):
         self.filters = []
 
     def get_transform_config(
-        self, data_access_factory: DataAccessFactory, statistics: ActorHandle, files: list[str]
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
     ) -> dict[str, Any]:
         """
         Set environment for transform execution

diff --git a/transforms/universal/fdedup/src/fdedup_transform.py b/transforms/universal/fdedup/src/fdedup_transform.py
@@ -19,7 +19,7 @@
 import numpy as np
 import pyarrow as pa
 import ray
-from data_processing.data_access import DataAccessFactory
+from data_processing.data_access import DataAccessFactoryBase
 from data_processing.ray import (
     DefaultTableTransformConfiguration,
     DefaultTableTransformRuntime,
@@ -334,7 +334,7 @@ def __init__(self, params: dict[str, Any]):
         self.random_delay_limit = self.params.get("random_delay_limit", 10)
 
     def get_transform_config(
-        self, data_access_factory: DataAccessFactory, statistics: ActorHandle, files: list[str]
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
     ) -> dict[str, Any]:
         """
         Set environment for filter execution
@@ -369,7 +369,7 @@ def get_transform_config(
         }
 
     def _create_doc_actors(
-        self, data_access_factory: DataAccessFactory, statistics: ActorHandle, files: list[str]
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, files: list[str]
     ) -> None:
         """
         Create document actors
@@ -420,7 +420,7 @@ def _create_doc_actors(
             )
 
     def _create_doc_actors_internal(
-        self, data_access_factory: DataAccessFactory, statistics: ActorHandle, mn_min_hash: MurmurMH, files: list[str]
+        self, data_access_factory: DataAccessFactoryBase, statistics: ActorHandle, mn_min_hash: MurmurMH, files: list[str]
     ) -> None:
         """
         Create document actors
@@ -495,7 +495,7 @@ def _create_doc_actors_internal(
 
     def _process_buckets(
         self,
-        data_access_factory: DataAccessFactory,
+        data_access_factory: DataAccessFactoryBase,
         statistics: ActorHandle,
         bucket_collectors: list[ActorHandle],
         minhash_collectors: list[ActorHandle],
@@ -591,7 +591,7 @@ def _process_buckets(
 
     def _preprocess_tables(
         self,
-        data_access_factory: DataAccessFactory,
+        data_access_factory: DataAccessFactoryBase,
         statistics: ActorHandle,
         files: list[str],
         mn_min_hash: MurmurMH,

diff --git a/transforms/universal/filter/src/filter_transform.py b/transforms/universal/filter/src/filter_transform.py
@@ -18,7 +18,7 @@
 import pyarrow as pa
 from data_processing.ray import DefaultTableTransformConfiguration, TransformLauncher
 from data_processing.transform import AbstractTableTransform
-from data_processing.utils import CLIArgumentProvider, ParamsUtils, get_logger
+from data_processing.utils import CLIArgumentProvider, get_logger
 
 
 logger = get_logger(__name__)