Skip to content
This repository
Browse code

use multiprocessing instead of parallelpython, even on windows

  • Loading branch information...
commit 58e3f0307490340ec8ba4f39e02a5e8d2aeec02b 1 parent 527eaca
Darren Dale authored December 18, 2011
48  doc/users/installation.rst
Source Rendered
@@ -4,19 +4,18 @@ Installation
4 4
 
5 5
 Praxes depends on a few python packages, most of which can be installed on
6 6
 Linux using the distribution's package manager, or on Mac OS X with MacPorts.
7  
-Note to windows users: you currently need to install the 32-bit python and
8  
-32-bit package installers, and you may need to run .exe installers as
9  
-administrator by right-clicking on them and choose "run as Administrator").
10  
-Check that you have the following installed:
11  
-
12  
-#. Python_ (version 2.7)
13  
-#. Cython_ (version 0.13 or later, only required for Mac OS X and Linux)
14  
-#. NumPy_ (version 1.5.1 or later)
15  
-#. PyQt4_ (version 4.5.2 or later) [#f1]_
16  
-#. PyMca_ (version 4.4.0 or later, Windows users, please see footnote) [#f2]_
17  
-#. matplotlib_ (0.98.3 or later)
18  
-#. h5py_ (1.3.0 or later) [#f3]_
19  
-#. parallelpython_ (1.6.1 or later, only required for Windows)
  7
+Note to windows users:  you may need to run .exe installers as administrator by
  8
+right-clicking on them and choose "run as Administrator"). Check that you have
  9
+the following installed:
  10
+
  11
+#. Python_ (version 2.6.x or 2.7.x. May already be installed on Linux)
  12
+#. Cython_ (version 0.15 or later, only required for Mac OS X and Linux)
  13
+#. NumPy_ (version 1.5.1 or later) [#f1]_
  14
+#. PyQt4_ (version 4.5.2 or later) [#f2]_
  15
+#. PyMca_ (version 4.4.0 or later) [#f3]_
  16
+#. matplotlib_ (1.1.0 or later) [#f1]_
  17
+#. h5py_ (2.1.0 or later) [#f1]_, [#f4]_
  18
+#. quantities_ (optional, only required by physref package)
20 19
 #. Praxes_
21 20
 
22 21
 To install Praxes on OS X or Linux, download the source tar.gz file, unpack it,
@@ -27,25 +26,28 @@ and run the following in the source directory::
27 26
 
28 27
 .. rubric:: Footnotes
29 28
 
30  
-.. [#f1] May require installing Qt_ on Mac, and development tools
  29
+
  30
+.. [#f1] Windows installers for 64-bit Python environments can be found
  31
+   `here <http://www.lfd.uci.edu/~gohlke/pythonlibs>`_
  32
+.. [#f2] May require installing Qt_ on Mac, and development tools
31 33
    like pyqt4-dev and pyqt4-dev-tools through the package manager on
32 34
    Linux.
33  
-.. [#f2] Windows users, please install
34  
-   the file that includes the python version in the name: e.g.
35  
-   PyMca-4.4.0.win32-py2.6.exe. Mac and linux users please install
36  
-   from source: e.g. pymca4.4.0-src.tgz.
37  
-.. [#f3] May require installing hdf5 on Linux and OS X, and development
  35
+.. [#f3] Mac and linux users please install from source: e.g.
  36
+   pymca4.4.1-src.tgz. Windows users should follow the `PythonPackages` link,
  37
+   and download the file that includes the platform and python version in the
  38
+   name: e.g. PyMca-4.4.1.win-amd64-py2.7.exe.
  39
+.. [#f4] May require installing hdf5 on Linux and OS X, and development
38 40
    libraries like libhdf5-dev if installing with a packager manager on
39  
-   some linux distributions. hdf5-1.8.4-patch1 is the recommended version.
  41
+   some linux distributions.
40 42
 
41 43
 
42  
-.. _Python: http://www.python.org/
  44
+.. _Python: http://www.python.org/download/releases/2.7.2/
43 45
 .. _Cython: http://pypi.python.org/pypi/Cython
44 46
 .. _NumPy: http://pypi.python.org/pypi/numpy
45 47
 .. _PyQt4: http://pypi.python.org/pypi/PyQt
46 48
 .. _Qt: http://qt.nokia.com/
47 49
 .. _matplotlib: http://pypi.python.org/pypi/matplotlib
48  
-.. _PyMca: http://pypi.python.org/pypi/PyMca
  50
+.. _PyMca: http://sourceforge.net/projects/pymca/files/pymca/PyMca4.4.1
49 51
 .. _h5py: http://pypi.python.org/pypi/h5py
  52
+.. _quantities: http://pypi.python.org/pypi/quantities
50 53
 .. _Praxes: http://github.com/praxes/praxes/downloads
51  
-.. _parallelpython: http://www.parallelpython.com
168  praxes/dispatch/pptaskmanager.py
... ...
@@ -1,168 +0,0 @@
1  
-"""
2  
-"""
3  
-from __future__ import with_statement
4  
-
5  
-import copy
6  
-import gc
7  
-import hashlib
8  
-import logging
9  
-import threading
10  
-import time
11  
-
12  
-import pp
13  
-#from PyQt4 import QtCore
14  
-import numpy as np
15  
-np.seterr(all='ignore')
16  
-
17  
-
18  
-logger = logging.getLogger(__file__)
19  
-DEBUG = False
20  
-
21  
-
22  
-class TaskManager(threading.Thread):
23  
-
24  
-    @property
25  
-    def job_server(self):
26  
-        return self._job_server
27  
-
28  
-    @property
29  
-    def lock(self):
30  
-        return self.__lock
31  
-
32  
-    @property
33  
-    def n_cpus(self):
34  
-        with self.lock:
35  
-            return copy.copy(self._n_cpus)
36  
-
37  
-    @property
38  
-    def n_points(self):
39  
-        with self.lock:
40  
-            return copy.copy(self._n_points)
41  
-
42  
-    @property
43  
-    def n_processed(self):
44  
-        with self.lock:
45  
-            return copy.copy(self._n_processed)
46  
-    @n_processed.setter
47  
-    def n_processed(self, val):
48  
-        with self.lock:
49  
-            self._n_processed = copy.copy(val)
50  
-
51  
-    @property
52  
-    def n_submitted(self):
53  
-        with self.lock:
54  
-            return copy.copy(self._n_submitted)
55  
-    @n_submitted.setter
56  
-    def n_submitted(self, val):
57  
-        with self.lock:
58  
-            self._n_submitted = copy.copy(val)
59  
-
60  
-    @property
61  
-    def scan(self):
62  
-        return self._scan
63  
-
64  
-    @property
65  
-    def stopped(self):
66  
-        with self.lock:
67  
-            return copy.copy(self.__stopped)
68  
-    @stopped.setter
69  
-    def stopped(self, val):
70  
-        with self.lock:
71  
-            self.__stopped = copy.copy(val)
72  
-
73  
-    def __init__(self, scan, progress_queue, **kwargs):
74  
-        super(TaskManager, self).__init__()
75  
-
76  
-        self.__lock = threading.RLock()
77  
-
78  
-        self._scan = scan
79  
-        self._n_points = scan.entry.npoints
80  
-
81  
-        self.progress_queue = progress_queue
82  
-        self.job_queue = []
83  
-
84  
-        self._job_server = pp.Server(ppservers=('*',))
85  
-
86  
-        n_cpus = self.job_server.get_ncpus()
87  
-        n_local_processes = kwargs.get('n_local_processes', n_cpus)
88  
-        self.job_server.set_ncpus(n_local_processes)
89  
-
90  
-        # total cpus, including local and remote:
91  
-        self._n_cpus = np.sum(
92  
-            [i for i in self.job_server.get_active_nodes().values()]
93  
-        )
94  
-
95  
-        self.__stopped = False
96  
-
97  
-        self._next_index = 0
98  
-        self._n_processed = 0
99  
-        self._n_submitted = 0
100  
-
101  
-    def __iter__(self):
102  
-        return self
103  
-
104  
-    def next(self):
105  
-        """
106  
-        This needs to be reimplemented to return either:
107  
-
108  
-        * (func, args) tuple such that func(*args) will work
109  
-        * 0 if the point was masked
110  
-        * None if the data is not yet available
111  
-        """
112  
-        raise NotImplementedError
113  
-
114  
-    def flush(self):
115  
-        #self.job_server.wait()
116  
-        while True:
117  
-            try:
118  
-                res = self.job_queue.pop(0)()
119  
-                if res is not None:
120  
-                    self.update_records(res)
121  
-            except IndexError:
122  
-                break
123  
-        self.n_submitted = 0
124  
-        self.queue_results()
125  
-
126  
-    def process_data(self):
127  
-        for item in self:
128  
-            if self.stopped:
129  
-                break
130  
-
131  
-            if item is None:
132  
-                # next data point is not yet available
133  
-                if self.n_submitted > 0:
134  
-                    self.flush()
135  
-                else:
136  
-                    time.sleep(0.1)
137  
-                continue
138  
-
139  
-            if item:
140  
-                f, args = item
141  
-                job = self.job_server.submit(
142  
-                    f, args, modules=("time", )
143  
-                    )
144  
-                self.job_queue.append(job)
145  
-
146  
-            self.n_processed += 1
147  
-            self.n_submitted += 1
148  
-
149  
-            if self.n_submitted >= self.n_cpus*3:
150  
-                self.flush()
151  
-
152  
-        if self.n_submitted > 0:
153  
-            self.flush()
154  
-
155  
-    def queue_results(self):
156  
-        stats = copy.deepcopy(self.job_server.get_stats())
157  
-        stats['n_processed'] = self.n_processed
158  
-        self.progress_queue.put(stats)
159  
-
160  
-    def run(self):
161  
-        self.process_data()
162  
-        self.scan.file.flush()
163  
-
164  
-    def stop(self):
165  
-        self.stopped = True
166  
-
167  
-    def update_records(self, data):
168  
-        raise NotImplementedError()
11  praxes/fluorescence/mcaanalysiswindow.py
@@ -342,10 +342,7 @@ def processComplete(self):
342 342
         self._results.flush()
343 343
 
344 344
     def processData(self):
345  
-        if sys.platform.startswith('win'):
346  
-            from .pptaskmanager import XfsTaskManager
347  
-        else:
348  
-            from .mptaskmanager import XfsTaskManager
  345
+        from .mptaskmanager import XfsTaskManager
349 346
 
350 347
         self.setMenuToolsActionsEnabled(False)
351 348
 
@@ -411,9 +408,11 @@ def setMenuToolsActionsEnabled(self, enabled=True):
411 408
 
412 409
     @classmethod
413 410
     def offersService(cls, h5Node):
414  
-        return isinstance(
  411
+        if isinstance(
415 412
             h5Node, (phynx.Entry, phynx.Measurement, phynx.MultiChannelAnalyzer)
416  
-        )
  413
+            ):
  414
+            return len(h5Node.entry.measurement.mcas) > 0
  415
+        return False
417 416
 
418 417
 
419 418
 #if __name__ == "__main__":
178  praxes/fluorescence/pptaskmanager.py
... ...
@@ -1,178 +0,0 @@
1  
-"""
2  
-"""
3  
-from __future__ import with_statement
4  
-
5  
-import copy
6  
-import logging
7  
-import sys
8  
-import time
9  
-
10  
-import numpy as np
11  
-import pp
12  
-from PyMca import ClassMcaTheory
13  
-from PyMca.ConcentrationsTool import ConcentrationsTool
14  
-import numpy as np
15  
-np.seterr(all='ignore')
16  
-
17  
-from praxes.dispatch.pptaskmanager import TaskManager
18  
-
19  
-
20  
-logger = logging.getLogger(__file__)
21  
-DEBUG = False
22  
-
23  
-def analyze_spectrum(index, spectrum, advanced_fit, mass_fraction_tool):
24  
-    start = time.time()
25  
-    advanced_fit.config['fit']['use_limit'] = 1
26  
-    # TODO: get the channels from the controller
27  
-    advanced_fit.setdata(y=spectrum)
28  
-    advanced_fit.estimate()
29  
-    estimate = time.time()
30  
-    if ('concentrations' in advanced_fit.config) and \
31  
-            (advanced_fit._fluoRates is None):
32  
-        fitresult, result = advanced_fit.startfit(digest=1)
33  
-    else:
34  
-        fitresult = advanced_fit.startfit(digest=0)
35  
-        result = advanced_fit.imagingDigestResult()
36  
-    result['index'] = index
37  
-    fit = time.time()
38  
-
39  
-    if mass_fraction_tool:
40  
-        temp = {}
41  
-        temp['fitresult'] = fitresult
42  
-        temp['result'] = result
43  
-        temp['result']['config'] = advanced_fit.config
44  
-        conc = mass_fraction_tool.processFitResult(
45  
-            fitresult=temp,
46  
-            elementsfrommatrix=False,
47  
-            fluorates=advanced_fit._fluoRates
48  
-            )
49  
-        result['concentrations'] = conc
50  
-    fitconc = time.time()
51  
-    report = {'estimate':estimate-start,
52  
-              'fit': fit-estimate,
53  
-              'fitconc': fitconc-fit}
54  
-
55  
-    return {
56  
-        'index': index,
57  
-        'result': result,
58  
-        'advanced_fit': advanced_fit,
59  
-        'report': report
60  
-    }
61  
-
62  
-
63  
-class XfsTaskManager(TaskManager):
64  
-
65  
-    @property
66  
-    def advanced_fit(self):
67  
-        with self.lock:
68  
-            return copy.deepcopy(self._advanced_fit)
69  
-    @advanced_fit.setter
70  
-    def advanced_fit(self, val):
71  
-        with self.lock:
72  
-            self._advanced_fit = val
73  
-
74  
-    @property
75  
-    def mass_fraction_tool(self):
76  
-        with self.lock:
77  
-            return copy.copy(self._mass_fraction_tool)
78  
-    @mass_fraction_tool.setter
79  
-    def mass_fraction_tool(self, val):
80  
-        with self.lock:
81  
-            self._mass_fraction_tool = val
82  
-
83  
-    def __init__(self, scan, config, progress_queue, **kwargs):
84  
-        super(XfsTaskManager, self).__init__(scan, progress_queue, **kwargs)
85  
-
86  
-        with scan:
87  
-            self._measurement = scan.entry.measurement
88  
-            self._indices = self._measurement.scalar_data['i']
89  
-            self._masked = self._measurement.masked
90  
-            try:
91  
-                # are we processing a group of mca elements...
92  
-                mcas = scan.mcas.values()
93  
-                self._counts = [mca['counts'] for mca in mcas]
94  
-                self._monitor = mcas[0].monitor.corrected_value
95  
-            except AttributeError:
96  
-                # or a single element?
97  
-                self._counts = [scan['counts']]
98  
-                self._monitor = scan.monitor.corrected_value
99  
-
100  
-        self._advanced_fit = ClassMcaTheory.McaTheory(config=config)
101  
-        self._advanced_fit.enableOptimizedLinearFit()
102  
-        self._mass_fraction_tool = None
103  
-        if 'concentrations' in config:
104  
-            self._mass_fraction_tool = ConcentrationsTool(config['concentrations'])
105  
-            self._mass_fraction_tool.config['time'] = 1
106  
-
107  
-    def next(self):
108  
-        i = self._next_index
109  
-        if i >= self.n_points:
110  
-            raise StopIteration()
111  
-
112  
-        with self.scan:
113  
-            try:
114  
-                suspect = i != self._indices[i]
115  
-            except IndexError:
116  
-                suspect = True
117  
-            if suspect:
118  
-                if i >= self._measurement.entry.npoints:
119  
-                    # entry.npoints has changed, scan was aborted
120  
-                    raise StopIteration()
121  
-                # expected the datapoint, but not yet acquired
122  
-                return None
123  
-
124  
-            self._next_index = i + 1
125  
-
126  
-            if self._masked[i]:
127  
-                return 0
128  
-
129  
-            cts = [counts.corrected_value[i] for counts in self._counts]
130  
-
131  
-            spectrum = np.sum(cts, 0)
132  
-            mft = self.mass_fraction_tool
133  
-            if self._monitor is not None:
134  
-                mft.config['flux'] = self._monitor[i]
135  
-        args = (
136  
-            i, spectrum, self.advanced_fit, self.mass_fraction_tool
137  
-            )
138  
-        return analyze_spectrum, args
139  
-
140  
-    def update_element_map(self, element, map_type, index, val):
141  
-        with self.scan:
142  
-            try:
143  
-                entry = '%s_%s'%(element, map_type)
144  
-                self.scan['element_maps'][entry][index] = val
145  
-            except ValueError:
146  
-                print "index %d out of range for %s" % (index, entry)
147  
-            except KeyError:
148  
-                print "%s not found in element_maps" % entry
149  
-            except TypeError:
150  
-                print entry, index, val
151  
-
152  
-    def update_records(self, data):
153  
-        if data is None:
154  
-            return
155  
-
156  
-        index = data['index']
157  
-        self.advanced_fit = data['advanced_fit']
158  
-
159  
-        result = data['result']
160  
-        for group in result['groups']:
161  
-            g = group.replace(' ', '_')
162  
-
163  
-            fit_area = result[group]['fitarea']
164  
-            if fit_area:
165  
-                sigma_area = result[group]['sigmaarea']/fit_area
166  
-            else:
167  
-                sigma_area = np.nan
168  
-
169  
-            self.update_element_map(g, 'fit', index, fit_area)
170  
-            self.update_element_map(g, 'fit_error', index, sigma_area)
171  
-
172  
-        try:
173  
-            mass_fractions = result['concentrations']['mass fraction']
174  
-            for key, val in mass_fractions.items():
175  
-                k = key.replace(' ', '_')
176  
-                self.update_element_map(k, 'mass_fraction', index, val)
177  
-        except KeyError:
178  
-            pass

0 notes on commit 58e3f03

Please sign in to comment.
Something went wrong with that request. Please try again.