Package fixes (#43)

adds github action to run tests on PR
adds two parameters to specify custom output files
restructures packages (again) because I noticed errors when trying to use this in our library
fixes pypi tests, they were not properly testing the installation from testpypi but simply using the local files

Co-authored-by: Ben Hurdelhey <Ben.Hurdelhey@vm-midea03.eaalab.hpi.uni-potsdam.de>
Co-authored-by: Christopher Hagedorn <christopher.schmidt1989@gmail.com>

.github/workflows/test-python.yml

@@ -0,0 +1,26 @@
+name: Run tests
+
+on:
+  pull_request:
+    branches: [master]
+
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.9"
+    - name: Install pytest
+      run: |
+        python3 -m pip install pytest --user
+    - name: Install dependencies
+      run: |
+        python3 -m pip install .
+    - name: Test with pytest
+      run: |
+        pytest --ignore test/test_manm_cs_pypi.py

.gitignore

@@ -138,3 +138,6 @@ data
 graph
 ground_truth*.gml
 samples*.csv
+data.csv
+graph.gml
+test_pypi

README.md

@@ -9,7 +9,7 @@ The related paper ["MANM-CS: Data Generation for Benchmarking Causal Structure L
 
 ### Installation / Usage via pip
 ```
-python3 -m pip install manm_cs
+python3 -m pip install manm-cs
 python3 -m manm_cs --num_nodes 10 --edge_density 0.5 --num_samples 10000 --discrete_node_ratio 0.5
 ```
 
@@ -60,10 +60,10 @@ python3 -m pip install .
 
 #### Execute data generation
 
-You can start the data generation with following command. The generated graph and the dataset are saved as ground_truth.gml and samples.csv in the current working directory. Available parameters for data generation can be seen with ```python3 -m src.manm_cs --help```.
+You can start the data generation with following command. The generated graph and the dataset are saved as ground_truth.gml and samples.csv in the current working directory. Available parameters for data generation can be seen with ```python3 -m manm_cs --help```.
 
 ```
-python3 -m src.manm_cs \
+python3 -m manm_cs \
     --num_nodes 10 \
     --edge_density 0.5 \
     --num_samples 10000 \

src/manm_cs/__main__.py → manm_cs/__main__.py

@@ -4,11 +4,11 @@
 
 import networkx as nx
 
-from src.manm_cs.graph import Graph, GraphBuilder
-from src.manm_cs.utils import write_single_csv
+from manm_cs.graph import Graph, GraphBuilder
+from manm_cs.utils import write_single_csv
 
-GROUND_TRUTH_FILE = "ground_truth.gml"
-SAMPLES_FILE = "samples.csv"
+GROUND_TRUTH_FILE = "ground_truth"
+SAMPLES_FILE = "samples"
 
 
 def type_in_range(type_: Type, lower_bound: Optional[float], upper_bound: Optional[float]) -> \
@@ -113,6 +113,12 @@ def parse_args():
                         required=False, default=1.0,
                         help='Defines the upper limit for beta values used for continuous parents. '
                         'Should be larger than lower_limit. Note that we sample from the union of [-upper,-lower] and [lower,upper]')
+    parser.add_argument('--output_ground_truth_file', type=str, required=False, default=GROUND_TRUTH_FILE,
+                    help='Output file (path) for the generated ground truth graph (gml). Relative to the directory from which the library is executed. '
+                    'Specify without file extension.')
+    parser.add_argument('--output_samples_file', type=str, required=False, default=SAMPLES_FILE,
+                    help='Output file (path) for the generated samples csv. Relative to the directory from which the library is executed.'
+                    'Specify without file extension.')
     args = parser.parse_args()
 
     assert args.min_discrete_value_classes <= args.max_discrete_value_classes, \
@@ -145,7 +151,7 @@ def graph_from_args(args) -> Graph:
     graph = graph_from_args(args)
 
     dfs = graph.sample(num_observations=args.num_samples, num_processes=args.num_processes)
-    write_single_csv(dataframes=dfs, target_path=SAMPLES_FILE)
+    write_single_csv(dataframes=dfs, target_path=f"{args.output_samples_file}.csv")
 
     nx_graph = graph.to_networkx_graph()
-    nx.write_gml(nx_graph, GROUND_TRUTH_FILE)
+    nx.write_gml(nx_graph, f"{args.output_ground_truth_file}.gml")

src/manm_cs/graph/graph.py → manm_cs/graph/graph.py

@@ -6,7 +6,7 @@
 import numpy as np
 from pathos.multiprocessing import ProcessingPool
 
-from src.manm_cs.variables.variable import Variable
+from manm_cs.variables.variable import Variable
 
 
 class Graph:

src/manm_cs/graph/graph_builder.py → manm_cs/graph/graph_builder.py

@@ -5,14 +5,14 @@
 import random
 from validation import validate_int, validate_float, validate_bool
 
-from src.manm_cs.graph import Graph
-from src.manm_cs.noise import GaussianNoiseBuilder, DiscreteNoiseBuilder
-from src.manm_cs.prob_distributions.continuous.bimodal_distribution import BimodalDistribution
-from src.manm_cs.prob_distributions.continuous.gaussian_distribution import GaussianDistribution
-from src.manm_cs.variables.continuous_variable import ContinuousVariable
-from src.manm_cs.variables.discrete_variable import DiscreteVariable
-from src.manm_cs.variables.variable import Variable
-from src.manm_cs.variables.variable_type import VariableType
+from manm_cs.graph import Graph
+from manm_cs.noise import GaussianNoiseBuilder, DiscreteNoiseBuilder
+from manm_cs.prob_distributions.continuous.bimodal_distribution import BimodalDistribution
+from manm_cs.prob_distributions.continuous.gaussian_distribution import GaussianDistribution
+from manm_cs.variables.continuous_variable import ContinuousVariable
+from manm_cs.variables.discrete_variable import DiscreteVariable
+from manm_cs.variables.variable import Variable
+from manm_cs.variables.variable_type import VariableType
 
 
 class GraphBuilder:

src/manm_cs/noise/continous_noise.py → manm_cs/noise/continous_noise.py

@@ -1,5 +1,5 @@
-from src.manm_cs.noise.noise import Noise
-from src.manm_cs.prob_distributions.continuous import ContinuousDistribution
+from manm_cs.noise.noise import Noise
+from manm_cs.prob_distributions.continuous import ContinuousDistribution
 
 
 class ContinuousNoise(Noise):

src/manm_cs/noise/discrete_noise.py → manm_cs/noise/discrete_noise.py

@@ -1,5 +1,5 @@
-from src.manm_cs.noise.noise import Noise
-from src.manm_cs.prob_distributions.discrete import DiscreteDistribution
+from manm_cs.noise.noise import Noise
+from manm_cs.prob_distributions.discrete import DiscreteDistribution
 
 
 class DiscreteNoise(Noise):

src/manm_cs/noise/discrete_noise_builder.py → manm_cs/noise/discrete_noise_builder.py

@@ -1,5 +1,5 @@
-from src.manm_cs.noise import DiscreteNoise
-from src.manm_cs.prob_distributions import CustomDiscreteDistribution
+from manm_cs.noise import DiscreteNoise
+from manm_cs.prob_distributions import CustomDiscreteDistribution
 
 
 class DiscreteNoiseBuilder:

src/manm_cs/noise/gaussian_noise_builder.py → manm_cs/noise/gaussian_noise_builder.py

@@ -1,5 +1,5 @@
-from src.manm_cs.noise import ContinuousNoise
-from src.manm_cs.prob_distributions.continuous import GaussianDistribution
+from manm_cs.noise import ContinuousNoise
+from manm_cs.prob_distributions.continuous import GaussianDistribution
 
 
 class GaussianNoiseBuilder:

src/manm_cs/noise/noise.py → manm_cs/noise/noise.py

@@ -3,10 +3,10 @@
 import numpy as np
 import pandas as pd
 
-from src.manm_cs.variables import VariableType
+from manm_cs.variables import VariableType
 
 if TYPE_CHECKING:
-    from src.manm_cs.prob_distributions.prob_distribution import ProbDistribution
+    from manm_cs.prob_distributions.prob_distribution import ProbDistribution
 
 
 class Noise:

manm_cs/prob_distributions/__init__.py

@@ -0,0 +1,4 @@
+from manm_cs.prob_distributions.continuous import GaussianDistribution
+from manm_cs.prob_distributions.discrete import BinomialDistribution
+from manm_cs.prob_distributions.discrete import CustomDiscreteDistribution
+from manm_cs.prob_distributions.discrete import UniformDiscreteDistribution

src/manm_cs/prob_distributions/continuous/bimodal_distribution.py → manm_cs/prob_distributions/continuous/bimodal_distribution.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from src.manm_cs.prob_distributions.continuous import ContinuousDistribution, GaussianDistribution
+from manm_cs.prob_distributions.continuous import ContinuousDistribution, GaussianDistribution
 
 
 class BimodalDistribution(ContinuousDistribution):

manm_cs/prob_distributions/continuous/continuous_distribution.py

@@ -0,0 +1,8 @@
+from abc import ABC
+
+from manm_cs.prob_distributions.prob_distribution import ProbDistribution
+from manm_cs.variables import VariableType
+
+
+class ContinuousDistribution(ProbDistribution, ABC):
+    type = VariableType.CONTINUOUS

src/manm_cs/prob_distributions/continuous/gaussian_distribution.py → manm_cs/prob_distributions/continuous/gaussian_distribution.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from src.manm_cs.prob_distributions.continuous import ContinuousDistribution
+from manm_cs.prob_distributions.continuous import ContinuousDistribution
 
 
 class GaussianDistribution(ContinuousDistribution):

src/manm_cs/prob_distributions/discrete/binomial_distribution.py → manm_cs/prob_distributions/discrete/binomial_distribution.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from src.manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
+from manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
 
 
 class BinomialDistribution(DiscreteDistribution):

src/manm_cs/prob_distributions/discrete/custom_discrete_distribution.py → manm_cs/prob_distributions/discrete/custom_discrete_distribution.py

@@ -3,7 +3,7 @@
 import numpy as np
 from numpy.random import random_sample
 
-from src.manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
+from manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
 
 
 class CustomDiscreteDistribution(DiscreteDistribution):

src/manm_cs/prob_distributions/discrete/discrete_distribution.py → manm_cs/prob_distributions/discrete/discrete_distribution.py

@@ -1,7 +1,7 @@
 from abc import ABC
 
-from src.manm_cs.prob_distributions.prob_distribution import ProbDistribution
-from src.manm_cs.variables import VariableType
+from manm_cs.prob_distributions.prob_distribution import ProbDistribution
+from manm_cs.variables import VariableType
 
 
 class DiscreteDistribution(ProbDistribution, ABC):

src/manm_cs/prob_distributions/discrete/random_discrete_distribution.py → manm_cs/prob_distributions/discrete/random_discrete_distribution.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from src.manm_cs.prob_distributions.discrete import CustomDiscreteDistribution
-from src.manm_cs.prob_distributions.discrete import DiscreteDistribution
+from manm_cs.prob_distributions.discrete import CustomDiscreteDistribution
+from manm_cs.prob_distributions.discrete import DiscreteDistribution
 
 
 class RandomDiscreteDistribution(DiscreteDistribution):

src/manm_cs/prob_distributions/discrete/uniform_discrete_distribution.py → manm_cs/prob_distributions/discrete/uniform_discrete_distribution.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from src.manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
+from manm_cs.prob_distributions.discrete.discrete_distribution import DiscreteDistribution
 
 
 class UniformDiscreteDistribution(DiscreteDistribution):

src/manm_cs/prob_distributions/prob_distribution.py → manm_cs/prob_distributions/prob_distribution.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from src.manm_cs.variables import VariableType
+from manm_cs.variables import VariableType
 
 
 class ProbDistribution(ABC):

src/manm_cs/variables/continuous_variable.py → manm_cs/variables/continuous_variable.py

@@ -3,8 +3,8 @@
 import numpy as np
 import pandas as pd
 
-from src.manm_cs.noise.continous_noise import ContinuousNoise
-from src.manm_cs.variables.variable import Variable, VariableType
+from manm_cs.noise.continous_noise import ContinuousNoise
+from manm_cs.variables.variable import Variable, VariableType
 
 
 class ContinuousVariable(Variable):

src/manm_cs/variables/discrete_variable.py → manm_cs/variables/discrete_variable.py

@@ -3,9 +3,9 @@
 import numpy as np
 import pandas as pd
 
-from src.manm_cs.noise.discrete_noise import DiscreteNoise
-from src.manm_cs.prob_distributions import CustomDiscreteDistribution
-from src.manm_cs.variables.variable import Variable, VariableType
+from manm_cs.noise.discrete_noise import DiscreteNoise
+from manm_cs.prob_distributions import CustomDiscreteDistribution
+from manm_cs.variables.variable import Variable, VariableType
 
 
 class DiscreteVariable(Variable):

src/manm_cs/variables/variable.py → manm_cs/variables/variable.py

@@ -3,8 +3,8 @@
 
 import pandas as pd
 
-from src.manm_cs.noise.noise import Noise
-from src.manm_cs.variables import VariableType
+from manm_cs.noise.noise import Noise
+from manm_cs.variables import VariableType
 
 
 class Variable(ABC):

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = manm_cs
-version = 0.0.5
+version = 0.1.0
 author = Johannes Huegle, Christopher Hagedorn, Lukas Boehme, Mats Poerschke, Jonas Umland
 author_email = johannes.huegle@hpi.de
 description = Data generation for causal structure learning based on mixed additive noise model (MANM)
@@ -16,18 +16,18 @@ classifiers =
 
 [options]
 package_dir =
-    = src
+    = .
 packages = find:
 python_requires = >=3.6
 install_requires =
-    pandas==1.1.5
-    numpy==1.19.5
-    multiprocess==0.70.11.1
+    pandas>=1.1.5
+    numpy>=1.19.5
+    multiprocess>=0.70.11.1
     validation==0.5.0
-    networkx==2.5
-    pathos==0.2.7
+    networkx>=2.5
+    pathos>=0.2.7
 
 
 [options.packages.find]
-where = src
+where = .
 

test/graph/test_graph.py

@@ -1,7 +1,7 @@
-from src.manm_cs.graph.graph import Graph
-from src.manm_cs.noise import DiscreteNoise
-from src.manm_cs.prob_distributions import BinomialDistribution, CustomDiscreteDistribution, UniformDiscreteDistribution
-from src.manm_cs.variables.discrete_variable import DiscreteVariable
+from manm_cs.graph.graph import Graph
+from manm_cs.noise import DiscreteNoise
+from manm_cs.prob_distributions import BinomialDistribution, CustomDiscreteDistribution, UniformDiscreteDistribution
+from manm_cs.variables.discrete_variable import DiscreteVariable
 import networkx as nx
 
 

test/test_manm_cs.py

@@ -0,0 +1,15 @@
+import os
+import time
+
+ground_truth = "graph"
+samples = "data"
+test_folder = "test_pypi"
+
+def test_manm_cs():
+    os.system(
+        "python3 -m pip install . &&" + # install dependencies from actual pip because they are not on testpypi
+        f"mkdir -p {test_folder} && cd {test_folder} && rm -f {samples} {ground_truth} &&" +
+        f"python3 -m manm_cs --num_nodes 10 --edge_density 0.5 --num_samples 10000 --discrete_node_ratio 0.5 --output_ground_truth_file \"{ground_truth}\" --output_samples_file \"{samples}\"")
+
+    assert os.path.isfile(test_folder + "/" + ground_truth + ".gml")
+    assert os.path.isfile(test_folder + "/" + samples + ".csv")

test/test_manm_cs_pypi.py

@@ -1,20 +1,18 @@
 import os
 import time
 
-ground_truth = "./ground_truth.gml"
-samples = "./samples.csv"
+ground_truth = "graph"
+samples = "data"
+test_folder = "test_pypi"
 
 def test_manm_cs_pypi():
 
-    if os.path.isfile(ground_truth):
-        os.rename(ground_truth, f"./ground_truth_old_{int(time.time())}.gml")
-    if os.path.isfile(samples):
-        os.rename(samples, f"./samples_old{int(time.time())}.csv")
-
     os.system(
         "python3 -m pip install . &&" + # install dependencies from actual pip because they are not on testpypi
+        f"mkdir -p {test_folder} && cd {test_folder} && rm -f {samples} {ground_truth} && " +
+        "python3 -m pip uninstall -y manm-cs &&" + # uninstall the package itself so we can install it from testpypi
         "python3 -m pip install -i https://test.pypi.org/simple/ manm-cs &&" + # install latest manm-cs from testpypi
-        "python3 -m manm_cs --num_nodes 10 --edge_density 0.5 --num_samples 10000 --discrete_node_ratio 0.5")
+        f"python3 -m manm_cs --num_nodes 10 --edge_density 0.5 --num_samples 10000 --discrete_node_ratio 0.5 --output_ground_truth_file \"{ground_truth}\" --output_samples_file \"{samples}\"")
 
-    assert os.path.isfile(ground_truth)
-    assert os.path.isfile(samples)
+    assert os.path.isfile(test_folder + "/" + ground_truth + ".gml")
+    assert os.path.isfile(test_folder + "/" + samples + ".csv")