Skip to content

Commit

Permalink
MAINT: Fixup 9cbc0bc
Browse files Browse the repository at this point in the history
- use `multiprocessing` in spawn mode
- finalise `Pool`
- remove `multiprocess` dependency
  • Loading branch information
ntfrgl committed Dec 22, 2023
1 parent 37a30c4 commit e3446bb
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 13 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ install:
- eval "$(conda shell.bash hook)"
- conda activate test-env
- travis_retry conda install -c conda-forge python=${TRAVIS_PYTHON_VERSION%-dev}
- travis_retry conda install -c conda-forge numpy scipy python-igraph h5netcdf multiprocess tqdm
- travis_retry conda install -c conda-forge numpy scipy python-igraph h5netcdf tqdm
- travis_retry conda update -c conda-forge --all

# testing dependencies
Expand Down
1 change: 0 additions & 1 deletion setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ install_requires =
scipy >= 1.10
igraph >= 0.10
h5netcdf >= 1.1
multiprocess >= 0.70
tqdm >= 4.66
python_requires = >=3.8
packages = find:
Expand Down
4 changes: 2 additions & 2 deletions src/pyunicorn/core/_ext/numerics.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -412,9 +412,9 @@ def _local_cliquishness_5thorder(
def _nsi_betweenness(
int N, ndarray[DWEIGHT_t, ndim=1] w,
ndarray[DEGREE_t, ndim=1] k,
ndarray[NODE_t, ndim=1] targets,
ndarray[NODE_t, ndim=1] flat_neighbors,
ndarray[MASK_t, ndim=1] is_source):
ndarray[MASK_t, ndim=1] is_source,
ndarray[NODE_t, ndim=1] targets):
"""
Performs Newman's algorithm. [Newman2001]_
"""
Expand Down
14 changes: 7 additions & 7 deletions src/pyunicorn/core/network.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@

import sys # performance testing
import time
from functools import wraps # helper function for decorators
from functools import partial, wraps
from multiprocessing import get_context, cpu_count

import numpy as np # array object and fast numerics
from numpy import random
Expand All @@ -35,7 +36,6 @@

import igraph # high performance graph theory tools

from multiprocess import Pool, cpu_count
from ..utils import mpi # parallelized computations

from ._ext.types import \
Expand Down Expand Up @@ -3323,15 +3323,15 @@ def nsi_betweenness(self, parallelize: bool = False, **kwargs):
assert k.sum() == len(flat_neighbors) == 2 * self.n_links
w, k = to_cy(w, DWEIGHT), to_cy(k, DEGREE)

def worker(batch):
return _nsi_betweenness(N, w, k, batch, flat_neighbors, is_source)

worker = partial(_nsi_betweenness, N, w, k, flat_neighbors, is_source)
if parallelize:
# (naively) parallelize loop over nodes
n_workers = cpu_count()
batches = np.array_split(to_cy(targets, NODE), n_workers)
pool = Pool() # pylint: disable=not-callable
betw_w = np.sum(pool.map(worker, batches), axis=0)
with get_context("spawn").Pool() as pool:
betw_w = np.sum(pool.map(worker, batches), axis=0)
pool.close()
pool.join()
else:
betw_w = worker(to_cy(targets, NODE))
return betw_w / w
Expand Down
6 changes: 4 additions & 2 deletions tests/test_core/test_network.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@
# L. Tupikina, V. Stolbova, R.V. Donner, N. Marwan, H.A. Dijkstra,
# and J. Kurths, "Unified functional network and nonlinear time series analysis
# for complex systems science: The pyunicorn package"

"""
Simple tests for the Network class.
"""

from functools import partial
from itertools import islice, product, repeat
from multiprocess import Pool, cpu_count
from multiprocessing import get_context, cpu_count

import pytest
import numpy as np
Expand Down Expand Up @@ -58,7 +60,7 @@ def compare_permutations(net, permutations, measures):
map(np.random.permutation, repeat(net.N, permutations))))
tasks = list(product(measures, range(permutations)))
cores = cpu_count()
with Pool() as pool: # pylint: disable=not-callable
with get_context("spawn").Pool() as pool:
pool.map(partial(compare_measures, net, pnets, rev_perms),
(list(islice(tasks, c, None, cores)) for c in range(cores)))
pool.close()
Expand Down

0 comments on commit e3446bb

Please sign in to comment.