Merge pull request #197 from ICRAR/move-io

Move io to dlg.data

daliuge-engine/dlg/apps/archiving.py

@@ -22,10 +22,9 @@
 import logging
 from ..drop import BarrierAppDROP, ContainerDROP
 from ..droputils import DROPFile
-from ..io import NgasIO, OpenMode, NgasLiteIO
+from dlg.data.io import NgasIO, OpenMode, NgasLiteIO
 from ..meta import (
     dlg_string_param,
-    dlg_float_param,
     dlg_int_param,
     dlg_component,
     dlg_batch_input,

daliuge-engine/dlg/apps/scp.py

@@ -25,9 +25,9 @@
     NullDROP,
     ContainerDROP,
 )
-from dlg.data.rdbms import RDBMSDrop
-from dlg.data.memory import InMemoryDROP, SharedMemoryDROP
-from dlg.data.ngas import NgasDROP
+from dlg.data.drops.rdbms import RDBMSDrop
+from dlg.data.drops.memory import InMemoryDROP, SharedMemoryDROP
+from dlg.data.drops.ngas import NgasDROP
 from dlg.meta import (
     dlg_string_param,
     dlg_float_param,

daliuge-engine/dlg/data/__init__.py

@@ -24,21 +24,3 @@
 DROPs that we have developed as examples and for real-life use. Most of them
 are based on the :class:`DataDROP`.
 """
-
-__all__ = [
-    "DirectoryContainer",
-    "FileDROP",
-    "InMemoryDROP",
-    "SharedMemoryDROP",
-    "NgasDROP",
-    "RDBMSDrop",
-    "PlasmaDROP",
-    "PlasmaFlightDROP",
-]
-
-from dlg.data.directorycontainer import DirectoryContainer
-from dlg.data.file import FileDROP
-from dlg.data.memory import InMemoryDROP, SharedMemoryDROP
-from dlg.data.ngas import NgasDROP
-from dlg.data.plasma import PlasmaDROP, PlasmaFlightDROP
-from dlg.data.rdbms import RDBMSDrop

daliuge-engine/dlg/data/drops/__init__.py

@@ -0,0 +1,44 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2015
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+"""
+This package contains several general-purpose data stores in form of
+DROPs that we have developed as examples and for real-life use. Most of them
+are based on the :class:`DataDROP`.
+"""
+
+__all__ = [
+    "DirectoryContainer",
+    "FileDROP",
+    "InMemoryDROP",
+    "SharedMemoryDROP",
+    "NgasDROP",
+    "RDBMSDrop",
+    "PlasmaDROP",
+    "PlasmaFlightDROP",
+]
+
+from dlg.data.drops.directorycontainer import DirectoryContainer
+from dlg.data.drops.file import FileDROP
+from dlg.data.drops.memory import InMemoryDROP, SharedMemoryDROP
+from dlg.data.drops.ngas import NgasDROP
+from dlg.data.drops.plasma import PlasmaDROP, PlasmaFlightDROP
+from dlg.data.drops.rdbms import RDBMSDrop

daliuge-engine/dlg/data/directorycontainer.py → daliuge-engine/dlg/data/drops/directorycontainer.py

@@ -23,7 +23,7 @@
 import os
 import shutil
 
-from dlg.data.file import FileDROP
+from dlg.data.drops.file import FileDROP
 from dlg.ddap_protocol import DROPRel, DROPLinkType
 from dlg.drop import PathBasedDrop, ContainerDROP
 from dlg.exceptions import InvalidDropException, InvalidRelationshipException

daliuge-engine/dlg/data/file.py → daliuge-engine/dlg/data/drops/file.py

@@ -27,7 +27,7 @@
 from dlg.ddap_protocol import DROPStates
 from dlg.drop import DataDROP, PathBasedDrop, logger, track_current_drop
 from dlg.exceptions import InvalidDropException
-from dlg.io import FileIO
+from dlg.data.io import FileIO
 from dlg.meta import dlg_bool_param
 from dlg.utils import isabs
 

daliuge-engine/dlg/data/json_drop.py → daliuge-engine/dlg/data/drops/json_drop.py

@@ -25,7 +25,7 @@
 import json
 import logging
 
-from dlg.data.file import FileDROP
+from dlg.data.drops.file import FileDROP
 
 LOG = logging.getLogger(__name__)
 

daliuge-engine/dlg/data/memory.py → daliuge-engine/dlg/data/drops/memory.py

@@ -28,7 +28,7 @@
 
 from dlg.common.reproducibility.reproducibility import common_hash
 from dlg.drop import DataDROP, logger
-from dlg.io import SharedMemoryIO, MemoryIO
+from dlg.data.io import SharedMemoryIO, MemoryIO
 
 
 ##

daliuge-engine/dlg/data/ngas.py → daliuge-engine/dlg/data/drops/ngas.py

@@ -21,7 +21,7 @@
 #
 from dlg.ddap_protocol import DROPStates
 from dlg.drop import DataDROP, logger, track_current_drop
-from dlg.io import NgasIO, NgasLiteIO
+from dlg.data.io import NgasIO, NgasLiteIO
 from dlg.meta import dlg_string_param, dlg_int_param
 
 

daliuge-engine/dlg/data/plasma.py → daliuge-engine/dlg/data/drops/plasma.py

@@ -26,7 +26,7 @@
 from pyarrow import plasma as plasma
 
 from dlg.drop import DataDROP
-from dlg.io import PlasmaIO, PlasmaFlightIO
+from dlg.data.io import PlasmaIO, PlasmaFlightIO
 from dlg.meta import dlg_string_param, dlg_bool_param
 
 

daliuge-engine/dlg/data/rdbms.py → daliuge-engine/dlg/data/drops/rdbms.py

@@ -24,7 +24,7 @@
 
 from dlg.drop import DataDROP, logger
 from dlg.exceptions import InvalidDropException
-from dlg.io import ErrorIO
+from dlg.data.io import ErrorIO
 from dlg.meta import dlg_dict_param
 from dlg.utils import prepare_sql
 

daliuge-engine/dlg/io.py → daliuge-engine/dlg/data/io.py

@@ -31,8 +31,8 @@
 from abc import abstractmethod, ABCMeta
 from typing import Optional, Union
 
-from . import ngaslite
-from .apps.plasmaflight import PlasmaFlightClient
+from dlg import ngaslite
+from dlg.apps.plasmaflight import PlasmaFlightClient
 
 import pyarrow
 import pyarrow.plasma as plasma

daliuge-engine/dlg/drop.py

@@ -58,7 +58,7 @@
 )
 from dlg.event import EventFirer
 from dlg.exceptions import InvalidDropException, InvalidRelationshipException
-from dlg.io import (
+from dlg.data.io import (
     DataIO,
     OpenMode,
     ErrorIO,

daliuge-engine/dlg/droputils.py

@@ -31,12 +31,12 @@
 import re
 import threading
 import traceback
-from typing import IO, Any, AsyncIterable, BinaryIO, Dict, Iterable, OrderedDict, Tuple, overload
+from typing import Any, Tuple
 import numpy as np
 
 from dlg.ddap_protocol import DROPStates
-from dlg.drop import AppDROP, AbstractDROP, DataDROP, PathBasedDrop
-from dlg.io import IOForURL, OpenMode
+from dlg.drop import AppDROP, AbstractDROP, DataDROP
+from dlg.data.io import IOForURL, OpenMode
 from dlg import common
 from dlg.common import DropType
 

daliuge-engine/dlg/environmentvar_drop.py

@@ -25,7 +25,7 @@
 import json
 
 from dlg.drop import AbstractDROP, DEFAULT_INTERNAL_PARAMETERS
-from dlg.io import MemoryIO
+from dlg.data.io import MemoryIO
 
 
 class KeyValueDROP:

daliuge-engine/dlg/graph_loader.py

@@ -41,11 +41,11 @@
     NullDROP,
     EndDROP,
 )
-from .data import *  # Make sure to grab every dataDROP
 from .environmentvar_drop import EnvironmentVarDROP
 from dlg.parset_drop import ParameterSetDROP
 from .exceptions import InvalidGraphException
-from dlg.data.json_drop import JsonDROP
+from dlg.data.drops.json_drop import JsonDROP
+from dlg.data.drops import *
 from .common import DropType
 
 

daliuge-engine/dlg/lifecycle/hsm/store.py

@@ -34,9 +34,9 @@
 
 import psutil
 
-from ...data.memory import InMemoryDROP
-from ...data.ngas import NgasDROP
-from ...data.file import FileDROP
+from dlg.data.drops.memory import InMemoryDROP
+from dlg.data.drops.ngas import NgasDROP
+from dlg.data.drops.file import FileDROP
 
 logger = logging.getLogger(__name__)
 

daliuge-engine/dlg/manager/node_manager.py

@@ -44,7 +44,7 @@
 from .. import rpc, utils
 from ..ddap_protocol import DROPStates
 from ..drop import AppDROP
-from ..data.memory import InMemoryDROP, SharedMemoryDROP
+from dlg.data.drops.memory import InMemoryDROP, SharedMemoryDROP
 from ..exceptions import (
     NoSessionException,
     SessionAlreadyExistsException,

daliuge-engine/dlg/parset_drop.py

@@ -24,7 +24,7 @@
 from abc import abstractmethod
 
 from dlg.drop import DataDROP, DEFAULT_INTERNAL_PARAMETERS
-from dlg.io import MemoryIO
+from dlg.data.io import MemoryIO
 from dlg.meta import dlg_string_param
 
 

daliuge-engine/dlg/s3_drop.py

@@ -26,7 +26,7 @@
 import botocore
 
 from .drop import AbstractDROP
-from .io import ErrorIO
+from dlg.data.io import ErrorIO
 from .meta import dlg_string_param, dlg_list_param
 
 

daliuge-engine/test/apps/test_bash.py

@@ -36,8 +36,8 @@
     StreamingInputOutputBashApp,
 )
 from dlg.ddap_protocol import DROPStates
-from dlg.data.memory import InMemoryDROP
-from dlg.data.file import FileDROP
+from dlg.data.drops.memory import InMemoryDROP
+from dlg.data.drops.file import FileDROP
 from dlg.droputils import DROPWaiterCtx
 
 

daliuge-engine/test/apps/test_crc.py

@@ -29,8 +29,8 @@
 from dlg import droputils
 from dlg.apps.crc import CRCApp, crc32
 from dlg.apps.dynlib import DynlibApp
-from dlg.data.memory import InMemoryDROP
-from dlg.data.file import FileDROP
+from dlg.data.drops.memory import InMemoryDROP
+from dlg.data.drops.file import FileDROP
 
 from . import test_dynlib
 from .setp_up import build_shared_library

daliuge-engine/test/apps/test_docker.py

@@ -31,8 +31,8 @@
 
 from dlg import droputils, utils
 from dlg.apps.dockerapp import DockerApp
-from dlg.data.ngas import NgasDROP
-from dlg.data.file import FileDROP
+from dlg.data.drops.ngas import NgasDROP
+from dlg.data.drops.file import FileDROP
 from dlg.droputils import DROPWaiterCtx
 
 docker_unavailable = True

daliuge-engine/test/apps/test_dynlib.py

@@ -25,14 +25,12 @@
 import time
 import unittest
 
-from .setp_up import build_shared_library
-from ..manager import test_dm
 from dlg import droputils
 from dlg.apps.dynlib import DynlibApp, DynlibStreamApp, DynlibProcApp
 from dlg.common import Categories
 from dlg.ddap_protocol import DROPRel, DROPLinkType, DROPStates
 from dlg.drop import NullDROP
-from dlg.data.memory import InMemoryDROP
+from dlg.data.drops.memory import InMemoryDROP
 
 from .setp_up import build_shared_library
 from ..manager import test_dm

daliuge-engine/test/apps/test_dynlib2.py

@@ -25,14 +25,12 @@
 import time
 import unittest
 
-from .setp_up import build_shared_library
-from ..manager import test_dm
 from dlg import droputils
 from dlg.apps.dynlib import DynlibApp, DynlibStreamApp, DynlibProcApp
 from dlg.common import Categories
 from dlg.ddap_protocol import DROPRel, DROPLinkType, DROPStates
 from dlg.drop import NullDROP
-from dlg.data.memory import InMemoryDROP
+from dlg.data.drops.memory import InMemoryDROP
 
 from .setp_up import build_shared_library
 from ..manager import test_dm

daliuge-engine/test/apps/test_pyfunc.py

@@ -25,16 +25,13 @@
 import pickle
 import random
 import unittest
-import pkg_resources
-import json
 import numpy
 
-from ..manager import test_dm
-from dlg import droputils, graph_loader
+from dlg import droputils
 from dlg.apps import pyfunc
 from dlg.common import Categories
 from dlg.ddap_protocol import DROPStates, DROPRel, DROPLinkType
-from dlg.data.memory import InMemoryDROP
+from dlg.data.drops.memory import InMemoryDROP
 from dlg.droputils import DROPWaiterCtx
 from dlg.exceptions import InvalidDropException
 

daliuge-engine/test/apps/test_simple.py

@@ -26,25 +26,22 @@
 import time
 import unittest
 from multiprocessing.pool import ThreadPool
-from numpy import random, mean, array, concatenate, random, testing
+from numpy import mean, array, concatenate, random, testing
 from psutil import cpu_count
 
 from dlg import droputils
 from dlg.apps.simple import (
     GenericScatterApp,
     GenericNpyScatterApp,
-    SleepApp,
-    CopyApp,
-    SleepAndCopyApp,
     ListAppendThrashingApp,
 )
 from dlg.apps.simple import RandomArrayApp, AverageArraysApp, HelloWorldApp
 from dlg.apps.simple import SleepApp, CopyApp, SleepAndCopyApp
 from dlg.ddap_protocol import DROPStates
 from dlg.drop import NullDROP
-from dlg.data.memory import InMemoryDROP
-from dlg.data.ngas import NgasDROP
-from dlg.data.file import FileDROP
+from dlg.data.drops.memory import InMemoryDROP
+from dlg.data.drops.ngas import NgasDROP
+from dlg.data.drops.file import FileDROP
 
 if sys.version_info >= (3, 8):
     from dlg.manager.shared_memory_manager import DlgSharedMemoryManager

daliuge-engine/test/apps/test_socket.py

@@ -26,7 +26,7 @@
 from dlg import utils
 from dlg.apps.socket_listener import SocketListenerApp
 from dlg.ddap_protocol import DROPStates
-from dlg.data.memory import InMemoryDROP
+from dlg.data.drops.memory import InMemoryDROP
 from dlg.droputils import DROPWaiterCtx
 from test.test_drop import SumupContainerChecksum
 

daliuge-engine/test/graphs/test_graphExecution.py

@@ -19,19 +19,15 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 #    MA 02111-1307  USA
 #
-from cmath import log
 import json
 import os
 import unittest
 
 from asyncio.log import logger
 import pkg_resources
 
-from dlg.data.memory import InMemoryDROP
-from dlg import runtime
+from dlg.data.drops.memory import InMemoryDROP
 from dlg import droputils
-from dlg import utils
-from dlg.ddap_protocol import DROPStates
 from dlg.manager.composite_manager import DataIslandManager
 from dlg.testutils import ManagerStarter