Merge pull request #316 from IBM/code2parquet

renaming of ingest_2_parquet

.make.defaults

@@ -331,6 +331,9 @@ __check_defined = \
 	$(MAKE) PIP_TARGET=data-prep-toolkit-spark .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_PYTHON_LIB_DIR) .defaults.install-src-venv;	\
 	$(MAKE) PYTHON_PROJECT_DIR=$(DPK_SPARK_LIB_DIR) .defaults.install-src-venv;	\
+	if [ -d ../python ]; then                                               \
+		$(MAKE) PYTHON_PROJECT_DIR=../python .defaults.install-src-venv;    \
+	fi
 	echo Installed source from Spark processing library for `which $(PYTHON)`
 
 # Run tests in test directory from that dir after adding ../src to PYTHONPATH

.make.versions

@@ -51,7 +51,12 @@ PROGLANG_SELECT_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 CODE_QUALITY_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 CODE_QUALITY_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 
-INGEST_TO_PARQUET_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+INPUT_CODE_TO_PARQUET_PYTHON_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+INPUT_CODE_TO_PARQUET_RAY_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
+
+
+
+
 INGEST_TO_PARQUET_VERSION=0.4.0$(RELEASE_VERSION_SUFFIX)
 
 

data-processing-lib/python/src/data_processing/transform/__init__.py

@@ -1,3 +1,4 @@
+from data_processing.transform.abstract_transform import AbstractTransform
 from data_processing.transform.binary_transform import AbstractBinaryTransform
 from data_processing.transform.table_transform import AbstractTableTransform
 from data_processing.transform.transform_statistics import TransformStatistics

data-processing-lib/python/src/data_processing/transform/transform_configuration.py

@@ -13,8 +13,7 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from data_processing.transform import AbstractBinaryTransform
-from data_processing.transform.abstract_transform import AbstractTransform
+from data_processing.transform import AbstractTransform
 from data_processing.utils import CLIArgumentProvider
 
 

data-processing-lib/ray/src/data_processing_ray/runtime/ray/transform_orchestrator.py

@@ -135,7 +135,10 @@ def orchestrate(
             "job_input_params": runtime_config.get_transform_metadata()
             | data_access_factory.get_input_params()
             | preprocessing_params.get_input_params(),
-            "execution_stats": resources | {"execution time, min": (time.time() - start_time)/60, },
+            "execution_stats": resources
+            | {
+                "execution time, min": (time.time() - start_time) / 60,
+            },
             "job_output_stats": stats,
         }
         logger.debug(f"Saving job metadata: {metadata}.")

scripts/kind/populate_minio.sh

@@ -12,8 +12,8 @@ mc mb kfp/test
 echo "copying data"
 # code modules
 mc cp --recursive ${REPOROOT}/transforms/code/code_quality/ray/test-data/input/ kfp/test/code_quality/input
-mc cp --recursive ${REPOROOT}/transforms/code/ingest_2_parquet/ray/test-data/input/data-processing-lib.zip kfp/test/ingest_2_parquet/input
-mc cp --recursive ${REPOROOT}/transforms/code/ingest_2_parquet/ray/test-data/languages/ kfp/test/ingest_2_parquet/languages
+mc cp --recursive ${REPOROOT}/transforms/code/inputcode2parquet/ray/test-data/input/data-processing-lib.zip kfp/test/inputcode2parquet/input
+mc cp --recursive ${REPOROOT}/transforms/code/inputcode2parquet/ray/test-data/languages/ kfp/test/inputcode2parquet/languages
 mc cp --recursive ${REPOROOT}/transforms/code/proglang_select/ray/test-data/input/ kfp/test/proglang_select/input
 mc cp --recursive ${REPOROOT}/transforms/code/proglang_select/ray/test-data/languages/ kfp/test/proglang_select/languages
 mc cp --recursive ${REPOROOT}/transforms/code/malware/ray/test-data/input/ kfp/test/malware/input