From 9d9fe711fbd4067475291f0b01cac8120f2310f1 Mon Sep 17 00:00:00 2001 From: stevenhua0320 Date: Sun, 19 Oct 2025 13:49:25 -0400 Subject: [PATCH 1/5] style: remove unused module, change style of string, fix codespell. --- src/diffpy/confutils/__init__.py | 4 +-- src/diffpy/confutils/tools.py | 18 ++++++------ src/diffpy/srxplanar/__init__.py | 6 ++-- src/diffpy/srxplanar/calculate.py | 46 +++++++++++++++++++------------ src/diffpy/srxplanar/loadimage.py | 14 ++++------ 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/src/diffpy/confutils/__init__.py b/src/diffpy/confutils/__init__.py index 301e567..3c3358e 100644 --- a/src/diffpy/confutils/__init__.py +++ b/src/diffpy/confutils/__init__.py @@ -14,9 +14,9 @@ ############################################################################## # some convenience imports -from diffpy.confutils.config import ConfigBase +from diffpy.confutils.config import ConfigBase # noqa: F401 # package version -from diffpy.srxplanar.version import __version__ +from diffpy.srxplanar.version import __version__ # noqa: F401 # End of file diff --git a/src/diffpy/confutils/tools.py b/src/diffpy/confutils/tools.py index aedd992..5df9a7c 100644 --- a/src/diffpy/confutils/tools.py +++ b/src/diffpy/confutils/tools.py @@ -49,9 +49,9 @@ def _configPropertyRW(name): read and write """ rv = property( - fget=lambda self: getattr(self.config, nm), - fset=lambda self, value: setattr(self.config, nm, value), - fdel=lambda self: delattr(self, nm), + fget=lambda self: getattr(self.config, name), + fset=lambda self, value: setattr(self.config, name, value), + fdel=lambda self: delattr(self, name), doc="attribute forwarded to self.config, read/write", ) return rv @@ -64,9 +64,9 @@ def str2bool(v): def opt2Str(opttype, optvalue): """Turn the value of one option to string, according to the option - type list of values are truned into "value1, value2, value3...". + type list of values are turned into "value1, value2, value3...". - :param opttype: string, type of opitons, for example 'str' or + :param opttype: string, type of options, for example 'str' or 'intlist' :param optvalue: value of the option :return: string, usually stored in ConfigBase.config @@ -105,7 +105,7 @@ def str2Opt(opttype, optvalue): """Convert the string to value of one option, according to the option type. - :param opttype: string, type of opitons, for example 'str' or + :param opttype: string, type of options, for example 'str' or 'intlist' :param optvalue: string, value of the option :return: value of the option, usually stored in ConfigBase.config @@ -113,7 +113,7 @@ def str2Opt(opttype, optvalue): # base converter conv = StrConv(opttype) if opttype.endswith("list"): - temp = re.split("\s*,\s*", optvalue) + temp = re.split(r"\s*,\s*", optvalue) rv = map(conv, temp) if len(temp) > 0 else [] else: rv = conv(optvalue) @@ -156,7 +156,7 @@ def checkCRC32(filename): """ try: fd = open(filename, "rb") - except: + except Exception: return "Read error" eachLine = fd.readline() prev = 0 @@ -175,7 +175,7 @@ def checkMD5(filename, blocksize=65536): """ try: fd = open(filename, "rb") - except: + except Exception: return "Read error" buf = fd.read(blocksize) md5 = hashlib.md5() diff --git a/src/diffpy/srxplanar/__init__.py b/src/diffpy/srxplanar/__init__.py index 71d8118..5c1e7bd 100644 --- a/src/diffpy/srxplanar/__init__.py +++ b/src/diffpy/srxplanar/__init__.py @@ -14,11 +14,11 @@ ############################################################################## # some convenience imports -from diffpy.srxplanar.srxplanar import SrXplanar -from diffpy.srxplanar.srxplanarconfig import SrXplanarConfig +from diffpy.srxplanar.srxplanar import SrXplanar # noqa: F401 +from diffpy.srxplanar.srxplanarconfig import SrXplanarConfig # noqa: F401 # package version -from diffpy.srxplanar.version import __version__ +from diffpy.srxplanar.version import __version__ # noqa: F401 # unit tests diff --git a/src/diffpy/srxplanar/calculate.py b/src/diffpy/srxplanar/calculate.py index 15eb964..7d2f7aa 100644 --- a/src/diffpy/srxplanar/calculate.py +++ b/src/diffpy/srxplanar/calculate.py @@ -15,8 +15,6 @@ import numpy as np import scipy.ndimage.filters as snf -import scipy.ndimage.morphology as snm -import scipy.sparse as ssp from diffpy.srxplanar.srxplanarconfig import _configPropertyR @@ -53,7 +51,8 @@ class Calculate(object): extracrop = _configPropertyR("extracrop") def __init__(self, p): - # create parameter proxy, so that parameters can be accessed by self.parametername in read-only mode + # create parameter proxy, so that parameters can be + # accessed by self.parametername in read-only mode self.config = p self.prepareCalculation() return @@ -106,7 +105,7 @@ def genIntegrationInds(self, mask=None): :return: self.bin_number """ self.maskedmatrix = np.array(self.tthorqmatrix) - if mask == None: + if mask is None: # mask = np.zeros((self.ydimension, self.xdimension), dtype=bool) mask = np.zeros((len(self.yr), len(self.xr)), dtype=bool) ce = self.cropedges @@ -114,8 +113,11 @@ def genIntegrationInds(self, mask=None): self.maskedmatrix[mask] = 1000.0 # extra crop - maskedmatrix = self.getMaskedmatrixPic() - # self.bin_number = np.array(np.histogram(maskedmatrix, self.bin_edges)[0], dtype=float) + self.getMaskedmatrixPic() + # self.bin_number = np.array( + # np.histogram(maskedmatrix, self.bin_edges)[0], + # dtype=float, + # ) # self.bin_number[self.bin_number <= 0] = 1 return # self.bin_number @@ -123,8 +125,8 @@ def intensity(self, pic): """2D to 1D image integration, intensity of pixels are binned and then take average, - :param pic: 2D array, array of raw counts, corrections hould be - already applied :retrun: 2d array, [tthorq, intensity, + :param pic: 2D array, array of raw counts, corrections should be + already applied :return: 2d array, [tthorq, intensity, unceratinty] or [tthorq, intensity] """ @@ -159,7 +161,7 @@ def getMaskedmatrixPic(self, pic=None): ) self.bin_number[self.bin_number <= 0] = 1 - if pic != None: + if pic is not None: ps = [max(s1, s2) for s1, s2 in zip(ce, ec)] rv = ( self.maskedmatrix[s[2] : s[3], s[0] : s[1]], @@ -171,7 +173,7 @@ def calculateIntensity(self, pic): """Calculate the 1D intensity. :param pic: 2D array, array of raw counts, raw counts should be - corrected :retrun: 1d array, 1D integrated intensity + corrected :return: 1d array, 1D integrated intensity """ maskedmatrix, pic = self.getMaskedmatrixPic(pic) @@ -182,8 +184,8 @@ def calculateIntensity(self, pic): def calculateVariance(self, pic): """Calculate the 1D intensity. - :param pic: 2D array, array of raw counts, corrections hould be - already applied :retrun: 1d array, variance of integrated + :param pic: 2D array, array of raw counts, corrections should be + already applied :return: 1d array, variance of integrated intensity """ maskedmatrix = self.getMaskedmatrixPic() @@ -196,9 +198,9 @@ def calculateVariance(self, pic): def calculateVarianceLocal(self, pic): """Calculate the variance of raw counts of each pixel are - calculated according to their loacl variance. + calculated according to their local variance. - :param pic: 2d array, 2d image array, corrections hould be + :param pic: 2d array, 2d image array, corrections should be already applied :return: 2d array, variance of each pixel """ @@ -251,7 +253,10 @@ def genTTHMatrix(self): sourceyr = self.distance * sint * sinr sourcezr = self.distance * cost - # tthmatrix1 = np.zeros((self.ydimension, self.xdimension), dtype=float) + # tthmatrix1 = np.zeros( + # (self.ydimension, self.xdimension), + # dtype=float, + # ) tthmatrix1 = np.zeros((len(self.yr), len(self.xr)), dtype=float) tthmatrix1 += ((-self.xr + sourcexr) * sourcexr).reshape( 1, len(self.xr) @@ -277,7 +282,10 @@ def genQMatrix(self): sourceyr = self.distance * sint * sinr sourcezr = self.distance * cost - # tthmatrix1 = np.zeros((self.ydimension, self.xdimension), dtype=float) + # tthmatrix1 = np.zeros( + # (self.ydimension, self.xdimension), + # dtype=float, + # ) tthmatrix1 = np.zeros((len(self.yr), len(self.xr)), dtype=float) tthmatrix1 += ((-self.xr + sourcexr) * sourcexr).reshape( 1, len(self.xr) @@ -322,7 +330,11 @@ def _polarizationCorrection(self): :return: 2d array, correction matrix to apply on the image """ if self.polcorrectionenable: - # tthmatrix = self.tthorqmatrix if self.integrationspace == 'twotheta' else self.genTTHMatrix() + # tthmatrix = ( + # self.tthorqmatrix + # if self.integrationspace == "twotheta" + # else self.genTTHMatrix() + # ) tthmatrix = self.tthmatrix azimuthmatrix = self.azimuthmatrix p = 0.5 * (1 + (np.cos(tthmatrix)) ** 2) diff --git a/src/diffpy/srxplanar/loadimage.py b/src/diffpy/srxplanar/loadimage.py index 010de43..7f44225 100644 --- a/src/diffpy/srxplanar/loadimage.py +++ b/src/diffpy/srxplanar/loadimage.py @@ -15,11 +15,9 @@ import fnmatch import os -import sys import time import numpy as np -from tifffile import imsave as saveImage from diffpy.srxplanar.srxplanarconfig import _configPropertyR @@ -30,7 +28,7 @@ def openImage(im): rv = fabio.openimage.openimage(im) return rv.data -except: +except ImportError: import tifffile def openImage(im): @@ -88,7 +86,7 @@ def loadImage(self, filename): else: image = openImage(filenamefull) i = 10 - except: + except FileNotFoundError: i = i + 1 time.sleep(0.5) image = self.flipImage(image) @@ -149,13 +147,13 @@ def genFileSet( file :return: set of str, a list of filenames """ - filenames = self.filenames if filenames == None else filenames - opendir = self.opendirectory if opendir == None else opendir + filenames = self.filenames if filenames is None else filenames + opendir = self.opendirectory if opendir is None else opendir includepattern = ( - self.includepattern if includepattern == None else includepattern + self.includepattern if includepattern is None else includepattern ) excludepattern = ( - self.excludepattern if excludepattern == None else excludepattern + self.excludepattern if excludepattern is None else excludepattern ) # filter the filenames according to include and exclude pattern filelist = os.listdir(opendir) From 23a074c1a7520fe22827ed95b570641416917dbf Mon Sep 17 00:00:00 2001 From: stevenhua0320 Date: Sun, 19 Oct 2025 13:59:52 -0400 Subject: [PATCH 2/5] style: pre-commit autofix to calculate.py --- src/diffpy/srxplanar/calculate.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/diffpy/srxplanar/calculate.py b/src/diffpy/srxplanar/calculate.py index 7d2f7aa..5106a02 100644 --- a/src/diffpy/srxplanar/calculate.py +++ b/src/diffpy/srxplanar/calculate.py @@ -126,8 +126,9 @@ def intensity(self, pic): and then take average, :param pic: 2D array, array of raw counts, corrections should be - already applied :return: 2d array, [tthorq, intensity, - unceratinty] or [tthorq, intensity] + already applied + :return: 2d array, [tthorq, intensity, unceratinty] or [tthorq, + intensity] """ intensity = self.calculateIntensity(pic) @@ -173,7 +174,8 @@ def calculateIntensity(self, pic): """Calculate the 1D intensity. :param pic: 2D array, array of raw counts, raw counts should be - corrected :return: 1d array, 1D integrated intensity + corrected + :return: 1d array, 1D integrated intensity """ maskedmatrix, pic = self.getMaskedmatrixPic(pic) @@ -185,8 +187,8 @@ def calculateVariance(self, pic): """Calculate the 1D intensity. :param pic: 2D array, array of raw counts, corrections should be - already applied :return: 1d array, variance of integrated - intensity + already applied + :return: 1d array, variance of integrated intensity """ maskedmatrix = self.getMaskedmatrixPic() From 90f0b36347244baee56dfb95df7f2efceb7e5bfd Mon Sep 17 00:00:00 2001 From: stevenhua0320 Date: Sun, 19 Oct 2025 22:22:14 -0400 Subject: [PATCH 3/5] style: delete unsued module to test if break functionality. --- src/diffpy/confutils/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/diffpy/confutils/__init__.py b/src/diffpy/confutils/__init__.py index 3c3358e..39cbde9 100644 --- a/src/diffpy/confutils/__init__.py +++ b/src/diffpy/confutils/__init__.py @@ -13,10 +13,7 @@ # ############################################################################## -# some convenience imports -from diffpy.confutils.config import ConfigBase # noqa: F401 - # package version -from diffpy.srxplanar.version import __version__ # noqa: F401 +from diffpy.srxplanar.version import __version__ # End of file From 105d4cdfe448ecb01f3d211130f1c4762e73e729 Mon Sep 17 00:00:00 2001 From: stevenhua0320 Date: Mon, 20 Oct 2025 13:38:33 -0400 Subject: [PATCH 4/5] style: delete file in confutils --- src/diffpy/confutils/__init__.py | 19 - src/diffpy/confutils/config.py | 823 ------------------------------- src/diffpy/confutils/tools.py | 209 -------- src/diffpy/confutils/version.py | 31 -- 4 files changed, 1082 deletions(-) delete mode 100644 src/diffpy/confutils/__init__.py delete mode 100644 src/diffpy/confutils/config.py delete mode 100644 src/diffpy/confutils/tools.py delete mode 100644 src/diffpy/confutils/version.py diff --git a/src/diffpy/confutils/__init__.py b/src/diffpy/confutils/__init__.py deleted file mode 100644 index 39cbde9..0000000 --- a/src/diffpy/confutils/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# diffpy.confutils by DANSE Diffraction group -# Simon J. L. Billinge -# (c) 2010 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## - -# package version -from diffpy.srxplanar.version import __version__ - -# End of file diff --git a/src/diffpy/confutils/config.py b/src/diffpy/confutils/config.py deleted file mode 100644 index 358a7fa..0000000 --- a/src/diffpy/confutils/config.py +++ /dev/null @@ -1,823 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# diffpy.confutils by DANSE Diffraction group -# Simon J. L. Billinge -# (c) 2012 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## -"""Package for organizing program configurations. It can read/write -configurations file, parse arguments from command lines, and also parse -arguments passed from method/function calling inside python. - -Note: for python 2.6, argparse and orderedDict is required, install them with -easy_install -""" - - -import argparse -import os -import re -import sys -from configparser import ConfigParser -from functools import partial - -try: - from collections import OrderedDict -except: - from ordereddict import OrderedDict - -from diffpy.confutils.tools import ( - FakeConfigFile, - StrConv, - _configPropertyR, - _configPropertyRad, - _configPropertyRW, - opt2Str, - str2bool, - str2Opt, -) - - -class ConfigBase(object): - """_optdatalist_default, _optdatalist are metadata used to - initialize the options, see below for examples. - - options presents in --help (in cmd), config file, headers have same order as - in these list, so arrange them in right order here. - - optional args to control if the options presents in args, config file or - file header - - 'args' - default is 'a' - if 'a', this option will be available in self.args - if 'n', this option will not be available in self.args - 'config' - default is 'a' - if 'f', this option will present in self.config and be written to - config file only in full mode - if 'a', this option will present in self.config and be written to - config file both in full and short mode - if 'n', this option will not present in self.config - 'header' - default is 'a' - if 'f', this option will be written to header only in full mode - if 'a', this option will be written to header both in full and short - mode - if 'n', this option will not be written to header - - so in short mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - """ - - # Text to display before the argument help - _description = """Description of configurations - """ - # Text to display after the argument help - _epilog = """ - """ - - """ - optdata contains these keys: - these args will be passed to argparse, see the documents of argparse for - detail information - - 'f': full, (positional) - 's': short - 'h': help - 't': type - 'a': action - 'n': nargs - 'd': default - 'c': choices - 'r': required - 'de': dest - 'co': const - """ - _optdatanamedict = { - "h": "help", - "t": "type", - "a": "action", - "n": "nargs", - "d": "default", - "c": "choices", - "r": "required", - "de": "dest", - "co": "const", - } - - # examples, overload it - _optdatalist_default = [ - [ - "configfile", - { - "sec": "Control", - "config": "f", - "header": "n", - "s": "c", - "h": "name of input config file", - "d": "", - }, - ], - [ - "createconfig", - { - "sec": "Control", - "config": "n", - "header": "n", - "h": "create a config file according to default or current values", - "d": "", - }, - ], - [ - "createconfigfull", - { - "sec": "Control", - "config": "n", - "header": "n", - "h": "create a full configurable config file", - "d": "", - }, - ], - ] - # examples, overload it - _optdatalist = [ - [ - "tifdirectory", - { - "sec": "Experiment", - "header": "n", - "s": "tifdir", - "h": "directory of raw tif files", - "d": "currentdir", - }, - ], - [ - "integrationspace", - { - "sec": "Experiment", - "h": "integration space, could be twotheta or qspace", - "d": "twotheta", - "c": ["twotheta", "qspace"], - }, - ], - [ - "wavelength", - { - "sec": "Experiment", - "h": "wavelength of x-ray, in A", - "d": 0.1000, - }, - ], - [ - "rotationd", - { - "sec": "Experiment", - "s": "rot", - "h": "rotation angle of tilt plane, in degree", - "d": 0.0, - }, - ], - [ - "includepattern", - { - "sec": "Beamline", - "s": "ipattern", - "h": "file name pattern for included files", - "n": "*", - "d": ["*.tif"], - }, - ], - [ - "excludepattern", - { - "sec": "Beamline", - "s": "epattern", - "h": "file name pattern for excluded files", - "n": "*", - "d": ["*.dark.tif", "*.raw.tif"], - }, - ], - [ - "fliphorizontal", - { - "sec": "Beamline", - "h": "filp the image horizontally", - "n": "?", - "co": True, - "d": False, - }, - ], - [ - "regulartmatrixenable", - { - "sec": "Others", - "h": "normalize tmatrix in splitting method", - "n": "?", - "co": True, - "d": False, - }, - ], - [ - "maskedges", - { - "sec": "Others", - "config": "f", - "header": "f", - "h": "mask the edge pixels, first four means the number of pixels masked in each edge \ - (left, right, top, bottom), the last one is the radius of a region masked around the corner", - "n": 5, - "d": [1, 1, 1, 1, 50], - }, - ], - ] - - # some default data - # configfile: default config file name - # headertitle: default title of header - _defaultdata = { - "configfile": ["config.cfg"], - "headertitle": "Configuration information", - } - - def __init__(self, filename=None, args=None, **kwargs): - """Init the class and update the values of options if specified - in filename/args/kwargs. - - it will: - 1. call self._preInit method - 2. find the config file if specified in filename/args/kwargs - if failed, try to find default config file - 3. update the options value using filename/args/kwargs - file > args > kwargs - - :param filename: str, file name of the config file - :param args: list of str, args passed from cmd - :param kwargs: dict, optional kwargs - - :return: None - """ - # call self._preInit - self._preInit(**kwargs) - - # update config, first detect if a default config should be load - filename = self._findDefaultConfigFile(filename, args, **kwargs) - rv = self.updateConfig(filename, args, **kwargs) - return - - # example, overload it - def _preInit(self, **kwargs): - """Method called in init process, overload it! - - this method will be called before reading config from - file/args/kwargs - """ - # for name in ['rotation']: - # setattr(self.__class__, name, _configPropertyRad(name+'d')) - # self._configlist['Experiment'].extend(['rotation']) - return - - ########################################################################### - - def _findConfigFile(self, filename=None, args=None, **kwargs): - """Find config file, if any config is specified in - filename/args/kwargs then return the filename of config. - - :param filename: str, file name of config file - :param filename: list of str, args passed from cmd - :param kwargs: optional kwargs - :return: name of config file if found, otherwise None - """ - rv = None - if filename != None: - rv = filename - if args != None: - if ("--configfile" in args) or ("-c" in args): - obj = self.args.parse_args(args) - rv = obj.configfile - if kwargs.has_key("configfile"): - rv = kwargs["configfile"] - return rv - - def _findDefaultConfigFile(self, filename=None, args=None, **kwargs): - """Find default config file, if any config is specified in - filename/args/kwargs or in self._defaultdata['configfile'], then - return the filename of config. - - kwargs > args > filename > default - - param filename: str, file name of config file - param filename: list of str, args passed from cmd - param kwargs: optional kwargs - - return: name of config file if found, otherwise None - """ - rv = self._findConfigFile(filename, args, **kwargs) - if rv == None: - for dconf in self._defaultdata["configfile"]: - if (os.path.exists(dconf)) and (rv == None): - rv = dconf - return rv - - ########################################################################### - - def _updateSelf(self, optnames=None, **kwargs): - """Update the options value, then copy the values in the - self.'options' to self.config. - - 1. call self._preUpdateSelf - 2. apply options' value from *self.option* to self.config - 3. call self._postUpdateSelf - - :param optnames: str or list of str, name of options whose value has - been changed, if None, update all options - """ - # so some check right here - self._preUpdateSelf(**kwargs) - # copy value to self.config - self._copySelftoConfig(optnames) - # so some check right here - self._postUpdateSelf(**kwargs) - return - - # example, overload it - def _preUpdateSelf(self, **kwargs): - """Additional process called in self._updateSelf, this method is - called before self._copySelftoConfig(), i.e. before copy options - value to self.config (config file)""" - return - - def _postUpdateSelf(self, **kwargs): - """Additional process called in self._updateSelf, this method is - called after self._copySelftoConfig(), i.e. before copy options - value to self.config (config file)""" - return - - ########################################################################### - - def _getTypeStr(self, optname): - """Return the type of option. - - :param optname: str, name of option - :return: string, type of the option - """ - opttype = self._getTypeStrC(optname) - return opttype - - @classmethod - def _getTypeStrC(cls, optname): - """Class method, return the type of option first try to get type - information from metadata, if failed, try to get type from - default value. - - :param optname: str, name of option - :return: string, type of the option - """ - optdata = cls._optdata[optname] - if optdata.has_key("t"): - opttype = optdata["t"] - else: - value = optdata["d"] - if isinstance(value, str): - opttype = "str" - elif isinstance(value, bool): - opttype = "bool" - elif isinstance(value, float): - opttype = "float" - elif isinstance(value, int): - opttype = "int" - elif isinstance(value, list): - if len(value) == 0: - opttype = "strlist" - elif isinstance(value[0], str): - opttype = "strlist" - elif isinstance(value[0], bool): - opttype = "boollist" - elif isinstance(value[0], float): - opttype = "floatlist" - elif isinstance(value[0], int): - opttype = "intlist" - - return opttype - - ########################################################################### - - def _detectAddSections(self): - """Detect sections present in self._optdata and add them to - self.config also add it to self._configlist.""" - self._detectAddSectionsC(self) - return - - @classmethod - def _detectAddSectionsC(cls): - """Class method, detect sections present in self._optdata and - add them to self.config also add it to self._configlist.""" - # seclist = [self._optdata[key]['sec'] for key in self._optdata.keys()] - seclist = [cls._optdata[opt[0]]["sec"] for opt in cls._optdatalist] - secdict = OrderedDict.fromkeys(seclist) - # for sec in set(seclist): - for sec in secdict.keys(): - cls.config.add_section(sec) - cls._configlist[sec] = [] - return - - def _addOpt(self, optname): - """Add options to self.config and self.args and self.*option*, - this will read metadata from self._optdatalist. - - :param optname: string, name of option - """ - self._addOptC(self, optname) - return - - @classmethod - def _addOptC(cls, optname): - """Class method, add options to self.config and self.args and - self.*option*, this will read metadata in self._optdatalist. - - :param optname: string, name of option - """ - optdata = cls._optdata[optname] - opttype = cls._getTypeStrC(optname) - - # replace currentdir in default to os.getcwd() - if optdata["d"] == "currentdir": - optdata["d"] = os.getcwd() - - # add to cls.'optname' - cls._addOptSelfC(optname, optdata) - - # add to cls.config - secname = optdata["sec"] if optdata.has_key("sec") else "Others" - cls._configlist[secname].append(optname) - if optdata.get("config", "a") != "n": - strvalue = ( - ", ".join(map(str, optdata["d"])) - if isinstance(optdata["d"], list) - else str(optdata["d"]) - ) - cls.config.set(secname, optname, strvalue) - # add to cls.args - if optdata.get("args", "a") != "n": - # transform optdata to a dict that can pass to add_argument method - pargs = dict() - for key in optdata.keys(): - if cls._optdatanamedict.has_key(key): - pargs[cls._optdatanamedict[key]] = optdata[key] - pargs["default"] = argparse.SUPPRESS - pargs["type"] = StrConv(opttype) - # add args - if optdata.has_key("f"): - cls.args.add_argument(optname, **pargs) - elif optdata.has_key("s"): - cls.args.add_argument( - "--" + optname, "-" + optdata["s"], **pargs - ) - else: - cls.args.add_argument("--" + optname, **pargs) - return - - @classmethod - def _addOptSelfC(cls, optname, optdata): - """Class method, assign options value to *self.option*, using - metadata. - - :param optname: string, name of the option - :param optdata: dict, metadata of the options, get it from - self._optdatalist - """ - setattr(cls, optname, optdata["d"]) - return - - def _copyConfigtoSelf(self, optnames=None): - """Copy the options' value from self.config to self.*option* - - :param optnames: str or list of str, names of options whose - value copied from self.config to self.*option*'. Set None to - update all - """ - if optnames != None: - optnames = optnames if isinstance(optnames, list) else [optnames] - else: - optnames = [] - for secname in self.config.sections(): - optnames += self.config.options(secname) - - for optname in optnames: - if self._optdata.has_key(optname): - secname = self._optdata[optname]["sec"] - opttype = self._getTypeStr(optname) - optvalue = self.config.get(secname, optname) - setattr(self, optname, str2Opt(opttype, optvalue)) - return - - def _copySelftoConfig(self, optnames=None): - """Copy the value from self.*option* to self.config. - - :param optname: str or list of str, names of options whose value - copied from self.*option* to self.config. Set None to update - all - """ - if optnames != None: - optnames = optnames if isinstance(optnames, list) else [optnames] - else: - optnames = [] - for secname in self.config.sections(): - optnames += self.config.options(secname) - - for optname in optnames: - if self._optdata.has_key(optname): - secname = self._optdata[optname]["sec"] - opttype = self._getTypeStr(optname) - optvalue = getattr(self, optname) - self.config.set(secname, optname, opt2Str(opttype, optvalue)) - return - - ########################################################################### - - def parseArgs(self, pargs): - """Parse args and update the value in self.*option*, this will - call the self.args() to parse args, - - :param pargs: list of string, arguments to parse, usually - comming from sys.argv - """ - obj = self.args.parse_args(pargs) - changedargs = obj.__dict__.keys() - for optname in changedargs: - if self._optdata.has_key(optname): - setattr(self, optname, getattr(obj, optname)) - # update self - if len(changedargs) > 0: - self._updateSelf(changedargs) - return obj - - def parseKwargs(self, **kwargs): - """Update self.*option* values according to the kwargs. - - :param kwargs: dict, keywords=value - """ - if kwargs != {}: - changedargs = [] - for optname, optvalue in kwargs.iteritems(): - if self._optdata.has_key(optname): - setattr(self, optname, optvalue) - changedargs.append(optname) - # update self - self._updateSelf(changedargs) - return - - def parseConfigFile(self, filename): - """Read a config file and update the self.*option* - - :param filename: str, file name of config file (include path) - """ - if filename != None: - filename = os.path.abspath(filename) - if os.path.exists(filename): - self.configfile = filename - self._copySelftoConfig() - fileobj = FakeConfigFile(filename) - # self.config.read(filename) - self.config.readfp(fileobj) - self._copyConfigtoSelf() - self._updateSelf() - return - - def updateConfig(self, filename=None, args=None, **kwargs): - """Update config according to config file, args(from sys.argv) - or **kwargs. - - 1. call self._preUpdateConfig() - 2. process file/args/kwargs passed to this method, - 3. read a configfile if specified in args or kwargs - 4. call self._postUpdateConfig() - 5. write config file if specified in args/kwargs - - :param filename: str, file name of the config file - :param args: list of str, args passed from cmd, - :param kwargs: dict, optional kwargs - - :return: True if anything updated, False if nothing updated - """ - # call self._preUpdateConfig - self._preUpdateConfig(**kwargs) - - filename = self._findConfigFile(filename, args, **kwargs) - if filename != None: - rv = self.parseConfigFile(filename) - if args != None: - rv = self.parseArgs(args) - if kwargs != {}: - rv = self.parseKwargs(**kwargs) - - if ( - (filename == None) - and ((args == None) or (args == [])) - and (kwargs == {}) - ): - rv = self._updateSelf() - - # call self._callbackUpdateConfig - self._postUpdateConfig(**kwargs) - - # write config file - self._createConfigFile() - return rv - - def _preUpdateConfig(self, **kwargs): - """Method called before parsing args or kwargs or config file, - in self.updateConfig.""" - return - - def _postUpdateConfig(self, **kwargs): - """Method called after parsing args or kwargs or config file, in - self.updateConfig.""" - return - - ########################################################################### - def _createConfigFile(self): - """Write output config file if specfied in configuration the - filename is specified by self.createconfig.""" - if (self.createconfig != "") and (self.createconfig != None): - self.writeConfig(self.createconfig, "short") - self.createconfig = "" - if (self.createconfigfull != "") and (self.createconfigfull != None): - self.writeConfig(self.createconfigfull, "full") - self.createconfigfull = "" - return - - def writeConfig(self, filename, mode="short", changeconfigfile=True): - """Write config to file. the file is compatiable with python - package ConfigParser. - - :param filename: string, name of file - :param mode: string, 'short' or 'full' ('s' or 'f'). in short - mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - """ - if changeconfigfile: - self.configfile = os.path.abspath(filename) - self._updateSelf() - # func decide if wirte the option to config according to mode - # options not present in self._optdata will not be written to config - if mode.startswith("s"): - mcond = ( - lambda optname: self._optdata.get( - optname, {"config": "n"} - ).get("config", "a") - == "a" - ) - else: - mcond = ( - lambda optname: self._optdata.get( - optname, {"config": "n"} - ).get("config", "a") - != "n" - ) - - lines = [] - for section in self.config._sections: - tlines = [] - for key, value in self.config._sections[section].items(): - if (key != "__name__") and mcond(key): - tlines.append( - "%s = %s" % (key, str(value).replace("\n", "\n\t")) - ) - if len(tlines) > 0: - lines.append("[%s]" % section) - lines.extend(tlines) - lines.append("") - rv = "\n".join(lines) + "\n" - fp = open(filename, "w") - fp.write(rv) - fp.close() - return - - def getHeader(self, title=None, mode="full"): - """Get a header of configurations values, - - :param title: str, title of header, if None, try to get it from - self.defaultvalue - :param mode: string, 'short' or 'full' ('s' or 'f'). in short - mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - :return: string, lines with line break that can be directly - writen to a text file - """ - - lines = [] - title = "# %s #" % ( - self._defaultdata["headertitle"] if title == None else title - ) - lines.append(title) - # func decide if wirte the option to header according to mode - # options not present in self._optdata will not be written to header - if mode.startswith("s"): - mcond = ( - lambda optname: self._optdata.get( - optname, {"header": "n"} - ).get("header", "a") - == "a" - ) - else: - mcond = ( - lambda optname: self._optdata.get( - optname, {"header": "n"} - ).get("header", "a") - != "n" - ) - - for secname in self._configlist.keys(): - tlines = [] - for optname in self._configlist[secname]: - if mcond(optname): - value = getattr(self, optname) - ttype = self._getTypeStr(optname) - strvalue = ( - ", ".join(map(str, value)) - if ttype.endswith("list") - else str(value) - ) - tlines.append("%s = %s" % (optname, strvalue)) - if len(tlines) > 0: - lines.append("[%s]" % secname) - lines.extend(tlines) - lines.append("") - rv = "\n".join(lines) + "\n" - return rv - - def resetDefault(self, optnames=None): - """Reset all values to their default value. - - :param optnames: list of str, name of options to reset, None for - all options - """ - if optnames == None: - optnames = self._optdata.keys() - for optname in optnames: - if self._optdata.has_key(optname): - setattr(self, optname, self._optdata[optname]["d"]) - self._updateSelf() - return - - ########################################################################### - # IMPORTANT call this method if you want to add options as class attributes!!! - - @classmethod - def initConfigClass(cls): - """Init config class and add options to class. - - IMPORTANT: call this method after you define the metadata of your config - class to add options as class attributes!!! - """ - cls._preInitConfigClass() - - cls.config = ConfigParser.ConfigParser(dict_type=OrderedDict) - cls.args = argparse.ArgumentParser( - description=cls._description, - epilog=cls._epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - cls._configlist = OrderedDict({}) - - cls._optdatalist = cls._optdatalist_default + cls._optdatalist - cls._optdata = dict(cls._optdatalist) - cls._detectAddSectionsC() - for opt in cls._optdatalist: - key = opt[0] - cls._addOptC(key) - - cls._postInitConfigClass() - return - - @classmethod - def _postInitConfigClass(cls): - """Additional processes called after initConfigClass. - - overload it - """ - pass - - @classmethod - def _preInitConfigClass(cls): - """Additional processes called before initConfigClass. - - overload it - """ - pass - - -# VERY IMPORTANT!!! -# add options to class -# initConfigClass(ConfigBase) -# ConfigBase.initConfigClass() - -if __name__ == "__main__": - - test = ConfigBase() - test.updateConfig() diff --git a/src/diffpy/confutils/tools.py b/src/diffpy/confutils/tools.py deleted file mode 100644 index 5df9a7c..0000000 --- a/src/diffpy/confutils/tools.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# diffpy.confutils by DANSE Diffraction group -# Simon J. L. Billinge -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## - -import hashlib -import re -import time -import zlib - -import numpy as np - - -def _configPropertyRad(nm): - """Helper function of options delegation, rad to degree.""" - rv = property( - fget=lambda self: np.radians(getattr(self, nm)), - fset=lambda self, val: setattr(self, nm, np.degrees(val)), - fdel=lambda self: delattr(self, nm), - ) - return rv - - -def _configPropertyR(name): - """Create a property that forwards self.name to self.config.name. - - read only - """ - rv = property( - fget=lambda self: getattr(self.config, name), - doc="attribute forwarded to self.config, read-only", - ) - return rv - - -def _configPropertyRW(name): - """Create a property that forwards self.name to self.config.name. - - read and write - """ - rv = property( - fget=lambda self: getattr(self.config, name), - fset=lambda self, value: setattr(self.config, name, value), - fdel=lambda self: delattr(self, name), - doc="attribute forwarded to self.config, read/write", - ) - return rv - - -def str2bool(v): - """Turn string to bool.""" - return v.lower() in ("yes", "true", "t", "1") - - -def opt2Str(opttype, optvalue): - """Turn the value of one option to string, according to the option - type list of values are turned into "value1, value2, value3...". - - :param opttype: string, type of options, for example 'str' or - 'intlist' - :param optvalue: value of the option - :return: string, usually stored in ConfigBase.config - """ - - if opttype.endswith("list"): - rv = ", ".join(map(str, optvalue)) - else: - rv = str(optvalue) - return rv - - -def StrConv(opttype): - """Get the type (a converter function) according to the opttype. - - the function doesn't take list - - :param opttype: string, a type of options, could be 'str', 'int', - 'float', or 'bool' - :return: type (converter function) - """ - if opttype.startswith("str"): - conv = str - elif opttype.startswith("int"): - conv = int - elif opttype.startswith("float"): - conv = float - elif opttype.startswith("bool"): - conv = str2bool - else: - conv = None - return conv - - -def str2Opt(opttype, optvalue): - """Convert the string to value of one option, according to the - option type. - - :param opttype: string, type of options, for example 'str' or - 'intlist' - :param optvalue: string, value of the option - :return: value of the option, usually stored in ConfigBase.config - """ - # base converter - conv = StrConv(opttype) - if opttype.endswith("list"): - temp = re.split(r"\s*,\s*", optvalue) - rv = map(conv, temp) if len(temp) > 0 else [] - else: - rv = conv(optvalue) - return rv - - -class FakeConfigFile(object): - """A fake configfile object used in reading config from header of - data or a real config file.""" - - def __init__(self, configfile, endline="###"): - self.configfile = configfile - self.fp = open(configfile) - self.endline = endline - self.ended = False - self.name = configfile - return - - def readline(self): - """Readline function.""" - line = self.fp.readline() - if line.startswith(self.endline) or self.ended: - rv = "" - self.ended = True - else: - rv = line - return rv - - def close(self): - """Close the file.""" - self.fp.close() - return - - -def checkCRC32(filename): - """Calculate the crc32 value of file. - - :param filename: path to the file - :return: crc32 value of file - """ - try: - fd = open(filename, "rb") - except Exception: - return "Read error" - eachLine = fd.readline() - prev = 0 - while eachLine: - prev = zlib.crc32(eachLine, prev) - eachLine = fd.readline() - fd.close() - return prev - - -def checkMD5(filename, blocksize=65536): - """Calculate the MD5 value of file. - - :param filename: path to the file - :return: md5 value of file - """ - try: - fd = open(filename, "rb") - except Exception: - return "Read error" - buf = fd.read(blocksize) - md5 = hashlib.md5() - while len(buf) > 0: - md5.update(buf) - buf = fd.read(blocksize) - fd.close() - return md5.hexdigest() - - -def checkFileVal(filename): - """Check file integrity using crc32 and md5. It will read file twice - then compare the crc32 and md5. If two results doesn't match, it - will wait until the file is completed written to disk. - - :param filename: path to the file - """ - valflag = False - lastcrc = checkCRC32(filename) - while not valflag: - currcrc = checkCRC32(filename) - if currcrc == lastcrc: - lastmd5 = checkMD5(filename) - time.sleep(0.01) - currmd5 = checkMD5(filename) - if lastmd5 == currmd5: - valflag = True - else: - time.sleep(0.5) - lastcrc = checkCRC32(filename) - return diff --git a/src/diffpy/confutils/version.py b/src/diffpy/confutils/version.py deleted file mode 100644 index 7aa6b57..0000000 --- a/src/diffpy/confutils/version.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# diffpy.confutils by DANSE Diffraction group -# Simon J. L. Billinge -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## -"""Definition of __version__, __date__, __gitsha__.""" - -from configparser import SafeConfigParser - -from pkg_resources import resource_stream - -# obtain version information from the version.cfg file -cp = SafeConfigParser() -cp.readfp(resource_stream(__name__, "version.cfg")) - -__version__ = cp.get("DEFAULT", "version") -__date__ = cp.get("DEFAULT", "date") -__gitsha__ = cp.get("DEFAULT", "commit") - -del cp - -# End of file From 506427397aa652ca9a6cc4188f6bb15960a68f88 Mon Sep 17 00:00:00 2001 From: stevenhua0320 Date: Mon, 20 Oct 2025 15:52:42 -0400 Subject: [PATCH 5/5] Revert "style: delete file in confutils" This reverts commit 105d4cdfe448ecb01f3d211130f1c4762e73e729. --- src/diffpy/confutils/__init__.py | 19 + src/diffpy/confutils/config.py | 823 +++++++++++++++++++++++++++++++ src/diffpy/confutils/tools.py | 209 ++++++++ src/diffpy/confutils/version.py | 31 ++ 4 files changed, 1082 insertions(+) create mode 100644 src/diffpy/confutils/__init__.py create mode 100644 src/diffpy/confutils/config.py create mode 100644 src/diffpy/confutils/tools.py create mode 100644 src/diffpy/confutils/version.py diff --git a/src/diffpy/confutils/__init__.py b/src/diffpy/confutils/__init__.py new file mode 100644 index 0000000..39cbde9 --- /dev/null +++ b/src/diffpy/confutils/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.confutils by DANSE Diffraction group +# Simon J. L. Billinge +# (c) 2010 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.txt for a list of people who contributed. +# See LICENSE.txt for license information. +# +############################################################################## + +# package version +from diffpy.srxplanar.version import __version__ + +# End of file diff --git a/src/diffpy/confutils/config.py b/src/diffpy/confutils/config.py new file mode 100644 index 0000000..358a7fa --- /dev/null +++ b/src/diffpy/confutils/config.py @@ -0,0 +1,823 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.confutils by DANSE Diffraction group +# Simon J. L. Billinge +# (c) 2012 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.txt for a list of people who contributed. +# See LICENSE.txt for license information. +# +############################################################################## +"""Package for organizing program configurations. It can read/write +configurations file, parse arguments from command lines, and also parse +arguments passed from method/function calling inside python. + +Note: for python 2.6, argparse and orderedDict is required, install them with +easy_install +""" + + +import argparse +import os +import re +import sys +from configparser import ConfigParser +from functools import partial + +try: + from collections import OrderedDict +except: + from ordereddict import OrderedDict + +from diffpy.confutils.tools import ( + FakeConfigFile, + StrConv, + _configPropertyR, + _configPropertyRad, + _configPropertyRW, + opt2Str, + str2bool, + str2Opt, +) + + +class ConfigBase(object): + """_optdatalist_default, _optdatalist are metadata used to + initialize the options, see below for examples. + + options presents in --help (in cmd), config file, headers have same order as + in these list, so arrange them in right order here. + + optional args to control if the options presents in args, config file or + file header + + 'args' - default is 'a' + if 'a', this option will be available in self.args + if 'n', this option will not be available in self.args + 'config' - default is 'a' + if 'f', this option will present in self.config and be written to + config file only in full mode + if 'a', this option will present in self.config and be written to + config file both in full and short mode + if 'n', this option will not present in self.config + 'header' - default is 'a' + if 'f', this option will be written to header only in full mode + if 'a', this option will be written to header both in full and short + mode + if 'n', this option will not be written to header + + so in short mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + """ + + # Text to display before the argument help + _description = """Description of configurations + """ + # Text to display after the argument help + _epilog = """ + """ + + """ + optdata contains these keys: + these args will be passed to argparse, see the documents of argparse for + detail information + + 'f': full, (positional) + 's': short + 'h': help + 't': type + 'a': action + 'n': nargs + 'd': default + 'c': choices + 'r': required + 'de': dest + 'co': const + """ + _optdatanamedict = { + "h": "help", + "t": "type", + "a": "action", + "n": "nargs", + "d": "default", + "c": "choices", + "r": "required", + "de": "dest", + "co": "const", + } + + # examples, overload it + _optdatalist_default = [ + [ + "configfile", + { + "sec": "Control", + "config": "f", + "header": "n", + "s": "c", + "h": "name of input config file", + "d": "", + }, + ], + [ + "createconfig", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": "create a config file according to default or current values", + "d": "", + }, + ], + [ + "createconfigfull", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": "create a full configurable config file", + "d": "", + }, + ], + ] + # examples, overload it + _optdatalist = [ + [ + "tifdirectory", + { + "sec": "Experiment", + "header": "n", + "s": "tifdir", + "h": "directory of raw tif files", + "d": "currentdir", + }, + ], + [ + "integrationspace", + { + "sec": "Experiment", + "h": "integration space, could be twotheta or qspace", + "d": "twotheta", + "c": ["twotheta", "qspace"], + }, + ], + [ + "wavelength", + { + "sec": "Experiment", + "h": "wavelength of x-ray, in A", + "d": 0.1000, + }, + ], + [ + "rotationd", + { + "sec": "Experiment", + "s": "rot", + "h": "rotation angle of tilt plane, in degree", + "d": 0.0, + }, + ], + [ + "includepattern", + { + "sec": "Beamline", + "s": "ipattern", + "h": "file name pattern for included files", + "n": "*", + "d": ["*.tif"], + }, + ], + [ + "excludepattern", + { + "sec": "Beamline", + "s": "epattern", + "h": "file name pattern for excluded files", + "n": "*", + "d": ["*.dark.tif", "*.raw.tif"], + }, + ], + [ + "fliphorizontal", + { + "sec": "Beamline", + "h": "filp the image horizontally", + "n": "?", + "co": True, + "d": False, + }, + ], + [ + "regulartmatrixenable", + { + "sec": "Others", + "h": "normalize tmatrix in splitting method", + "n": "?", + "co": True, + "d": False, + }, + ], + [ + "maskedges", + { + "sec": "Others", + "config": "f", + "header": "f", + "h": "mask the edge pixels, first four means the number of pixels masked in each edge \ + (left, right, top, bottom), the last one is the radius of a region masked around the corner", + "n": 5, + "d": [1, 1, 1, 1, 50], + }, + ], + ] + + # some default data + # configfile: default config file name + # headertitle: default title of header + _defaultdata = { + "configfile": ["config.cfg"], + "headertitle": "Configuration information", + } + + def __init__(self, filename=None, args=None, **kwargs): + """Init the class and update the values of options if specified + in filename/args/kwargs. + + it will: + 1. call self._preInit method + 2. find the config file if specified in filename/args/kwargs + if failed, try to find default config file + 3. update the options value using filename/args/kwargs + file > args > kwargs + + :param filename: str, file name of the config file + :param args: list of str, args passed from cmd + :param kwargs: dict, optional kwargs + + :return: None + """ + # call self._preInit + self._preInit(**kwargs) + + # update config, first detect if a default config should be load + filename = self._findDefaultConfigFile(filename, args, **kwargs) + rv = self.updateConfig(filename, args, **kwargs) + return + + # example, overload it + def _preInit(self, **kwargs): + """Method called in init process, overload it! + + this method will be called before reading config from + file/args/kwargs + """ + # for name in ['rotation']: + # setattr(self.__class__, name, _configPropertyRad(name+'d')) + # self._configlist['Experiment'].extend(['rotation']) + return + + ########################################################################### + + def _findConfigFile(self, filename=None, args=None, **kwargs): + """Find config file, if any config is specified in + filename/args/kwargs then return the filename of config. + + :param filename: str, file name of config file + :param filename: list of str, args passed from cmd + :param kwargs: optional kwargs + :return: name of config file if found, otherwise None + """ + rv = None + if filename != None: + rv = filename + if args != None: + if ("--configfile" in args) or ("-c" in args): + obj = self.args.parse_args(args) + rv = obj.configfile + if kwargs.has_key("configfile"): + rv = kwargs["configfile"] + return rv + + def _findDefaultConfigFile(self, filename=None, args=None, **kwargs): + """Find default config file, if any config is specified in + filename/args/kwargs or in self._defaultdata['configfile'], then + return the filename of config. + + kwargs > args > filename > default + + param filename: str, file name of config file + param filename: list of str, args passed from cmd + param kwargs: optional kwargs + + return: name of config file if found, otherwise None + """ + rv = self._findConfigFile(filename, args, **kwargs) + if rv == None: + for dconf in self._defaultdata["configfile"]: + if (os.path.exists(dconf)) and (rv == None): + rv = dconf + return rv + + ########################################################################### + + def _updateSelf(self, optnames=None, **kwargs): + """Update the options value, then copy the values in the + self.'options' to self.config. + + 1. call self._preUpdateSelf + 2. apply options' value from *self.option* to self.config + 3. call self._postUpdateSelf + + :param optnames: str or list of str, name of options whose value has + been changed, if None, update all options + """ + # so some check right here + self._preUpdateSelf(**kwargs) + # copy value to self.config + self._copySelftoConfig(optnames) + # so some check right here + self._postUpdateSelf(**kwargs) + return + + # example, overload it + def _preUpdateSelf(self, **kwargs): + """Additional process called in self._updateSelf, this method is + called before self._copySelftoConfig(), i.e. before copy options + value to self.config (config file)""" + return + + def _postUpdateSelf(self, **kwargs): + """Additional process called in self._updateSelf, this method is + called after self._copySelftoConfig(), i.e. before copy options + value to self.config (config file)""" + return + + ########################################################################### + + def _getTypeStr(self, optname): + """Return the type of option. + + :param optname: str, name of option + :return: string, type of the option + """ + opttype = self._getTypeStrC(optname) + return opttype + + @classmethod + def _getTypeStrC(cls, optname): + """Class method, return the type of option first try to get type + information from metadata, if failed, try to get type from + default value. + + :param optname: str, name of option + :return: string, type of the option + """ + optdata = cls._optdata[optname] + if optdata.has_key("t"): + opttype = optdata["t"] + else: + value = optdata["d"] + if isinstance(value, str): + opttype = "str" + elif isinstance(value, bool): + opttype = "bool" + elif isinstance(value, float): + opttype = "float" + elif isinstance(value, int): + opttype = "int" + elif isinstance(value, list): + if len(value) == 0: + opttype = "strlist" + elif isinstance(value[0], str): + opttype = "strlist" + elif isinstance(value[0], bool): + opttype = "boollist" + elif isinstance(value[0], float): + opttype = "floatlist" + elif isinstance(value[0], int): + opttype = "intlist" + + return opttype + + ########################################################################### + + def _detectAddSections(self): + """Detect sections present in self._optdata and add them to + self.config also add it to self._configlist.""" + self._detectAddSectionsC(self) + return + + @classmethod + def _detectAddSectionsC(cls): + """Class method, detect sections present in self._optdata and + add them to self.config also add it to self._configlist.""" + # seclist = [self._optdata[key]['sec'] for key in self._optdata.keys()] + seclist = [cls._optdata[opt[0]]["sec"] for opt in cls._optdatalist] + secdict = OrderedDict.fromkeys(seclist) + # for sec in set(seclist): + for sec in secdict.keys(): + cls.config.add_section(sec) + cls._configlist[sec] = [] + return + + def _addOpt(self, optname): + """Add options to self.config and self.args and self.*option*, + this will read metadata from self._optdatalist. + + :param optname: string, name of option + """ + self._addOptC(self, optname) + return + + @classmethod + def _addOptC(cls, optname): + """Class method, add options to self.config and self.args and + self.*option*, this will read metadata in self._optdatalist. + + :param optname: string, name of option + """ + optdata = cls._optdata[optname] + opttype = cls._getTypeStrC(optname) + + # replace currentdir in default to os.getcwd() + if optdata["d"] == "currentdir": + optdata["d"] = os.getcwd() + + # add to cls.'optname' + cls._addOptSelfC(optname, optdata) + + # add to cls.config + secname = optdata["sec"] if optdata.has_key("sec") else "Others" + cls._configlist[secname].append(optname) + if optdata.get("config", "a") != "n": + strvalue = ( + ", ".join(map(str, optdata["d"])) + if isinstance(optdata["d"], list) + else str(optdata["d"]) + ) + cls.config.set(secname, optname, strvalue) + # add to cls.args + if optdata.get("args", "a") != "n": + # transform optdata to a dict that can pass to add_argument method + pargs = dict() + for key in optdata.keys(): + if cls._optdatanamedict.has_key(key): + pargs[cls._optdatanamedict[key]] = optdata[key] + pargs["default"] = argparse.SUPPRESS + pargs["type"] = StrConv(opttype) + # add args + if optdata.has_key("f"): + cls.args.add_argument(optname, **pargs) + elif optdata.has_key("s"): + cls.args.add_argument( + "--" + optname, "-" + optdata["s"], **pargs + ) + else: + cls.args.add_argument("--" + optname, **pargs) + return + + @classmethod + def _addOptSelfC(cls, optname, optdata): + """Class method, assign options value to *self.option*, using + metadata. + + :param optname: string, name of the option + :param optdata: dict, metadata of the options, get it from + self._optdatalist + """ + setattr(cls, optname, optdata["d"]) + return + + def _copyConfigtoSelf(self, optnames=None): + """Copy the options' value from self.config to self.*option* + + :param optnames: str or list of str, names of options whose + value copied from self.config to self.*option*'. Set None to + update all + """ + if optnames != None: + optnames = optnames if isinstance(optnames, list) else [optnames] + else: + optnames = [] + for secname in self.config.sections(): + optnames += self.config.options(secname) + + for optname in optnames: + if self._optdata.has_key(optname): + secname = self._optdata[optname]["sec"] + opttype = self._getTypeStr(optname) + optvalue = self.config.get(secname, optname) + setattr(self, optname, str2Opt(opttype, optvalue)) + return + + def _copySelftoConfig(self, optnames=None): + """Copy the value from self.*option* to self.config. + + :param optname: str or list of str, names of options whose value + copied from self.*option* to self.config. Set None to update + all + """ + if optnames != None: + optnames = optnames if isinstance(optnames, list) else [optnames] + else: + optnames = [] + for secname in self.config.sections(): + optnames += self.config.options(secname) + + for optname in optnames: + if self._optdata.has_key(optname): + secname = self._optdata[optname]["sec"] + opttype = self._getTypeStr(optname) + optvalue = getattr(self, optname) + self.config.set(secname, optname, opt2Str(opttype, optvalue)) + return + + ########################################################################### + + def parseArgs(self, pargs): + """Parse args and update the value in self.*option*, this will + call the self.args() to parse args, + + :param pargs: list of string, arguments to parse, usually + comming from sys.argv + """ + obj = self.args.parse_args(pargs) + changedargs = obj.__dict__.keys() + for optname in changedargs: + if self._optdata.has_key(optname): + setattr(self, optname, getattr(obj, optname)) + # update self + if len(changedargs) > 0: + self._updateSelf(changedargs) + return obj + + def parseKwargs(self, **kwargs): + """Update self.*option* values according to the kwargs. + + :param kwargs: dict, keywords=value + """ + if kwargs != {}: + changedargs = [] + for optname, optvalue in kwargs.iteritems(): + if self._optdata.has_key(optname): + setattr(self, optname, optvalue) + changedargs.append(optname) + # update self + self._updateSelf(changedargs) + return + + def parseConfigFile(self, filename): + """Read a config file and update the self.*option* + + :param filename: str, file name of config file (include path) + """ + if filename != None: + filename = os.path.abspath(filename) + if os.path.exists(filename): + self.configfile = filename + self._copySelftoConfig() + fileobj = FakeConfigFile(filename) + # self.config.read(filename) + self.config.readfp(fileobj) + self._copyConfigtoSelf() + self._updateSelf() + return + + def updateConfig(self, filename=None, args=None, **kwargs): + """Update config according to config file, args(from sys.argv) + or **kwargs. + + 1. call self._preUpdateConfig() + 2. process file/args/kwargs passed to this method, + 3. read a configfile if specified in args or kwargs + 4. call self._postUpdateConfig() + 5. write config file if specified in args/kwargs + + :param filename: str, file name of the config file + :param args: list of str, args passed from cmd, + :param kwargs: dict, optional kwargs + + :return: True if anything updated, False if nothing updated + """ + # call self._preUpdateConfig + self._preUpdateConfig(**kwargs) + + filename = self._findConfigFile(filename, args, **kwargs) + if filename != None: + rv = self.parseConfigFile(filename) + if args != None: + rv = self.parseArgs(args) + if kwargs != {}: + rv = self.parseKwargs(**kwargs) + + if ( + (filename == None) + and ((args == None) or (args == [])) + and (kwargs == {}) + ): + rv = self._updateSelf() + + # call self._callbackUpdateConfig + self._postUpdateConfig(**kwargs) + + # write config file + self._createConfigFile() + return rv + + def _preUpdateConfig(self, **kwargs): + """Method called before parsing args or kwargs or config file, + in self.updateConfig.""" + return + + def _postUpdateConfig(self, **kwargs): + """Method called after parsing args or kwargs or config file, in + self.updateConfig.""" + return + + ########################################################################### + def _createConfigFile(self): + """Write output config file if specfied in configuration the + filename is specified by self.createconfig.""" + if (self.createconfig != "") and (self.createconfig != None): + self.writeConfig(self.createconfig, "short") + self.createconfig = "" + if (self.createconfigfull != "") and (self.createconfigfull != None): + self.writeConfig(self.createconfigfull, "full") + self.createconfigfull = "" + return + + def writeConfig(self, filename, mode="short", changeconfigfile=True): + """Write config to file. the file is compatiable with python + package ConfigParser. + + :param filename: string, name of file + :param mode: string, 'short' or 'full' ('s' or 'f'). in short + mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + """ + if changeconfigfile: + self.configfile = os.path.abspath(filename) + self._updateSelf() + # func decide if wirte the option to config according to mode + # options not present in self._optdata will not be written to config + if mode.startswith("s"): + mcond = ( + lambda optname: self._optdata.get( + optname, {"config": "n"} + ).get("config", "a") + == "a" + ) + else: + mcond = ( + lambda optname: self._optdata.get( + optname, {"config": "n"} + ).get("config", "a") + != "n" + ) + + lines = [] + for section in self.config._sections: + tlines = [] + for key, value in self.config._sections[section].items(): + if (key != "__name__") and mcond(key): + tlines.append( + "%s = %s" % (key, str(value).replace("\n", "\n\t")) + ) + if len(tlines) > 0: + lines.append("[%s]" % section) + lines.extend(tlines) + lines.append("") + rv = "\n".join(lines) + "\n" + fp = open(filename, "w") + fp.write(rv) + fp.close() + return + + def getHeader(self, title=None, mode="full"): + """Get a header of configurations values, + + :param title: str, title of header, if None, try to get it from + self.defaultvalue + :param mode: string, 'short' or 'full' ('s' or 'f'). in short + mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + :return: string, lines with line break that can be directly + writen to a text file + """ + + lines = [] + title = "# %s #" % ( + self._defaultdata["headertitle"] if title == None else title + ) + lines.append(title) + # func decide if wirte the option to header according to mode + # options not present in self._optdata will not be written to header + if mode.startswith("s"): + mcond = ( + lambda optname: self._optdata.get( + optname, {"header": "n"} + ).get("header", "a") + == "a" + ) + else: + mcond = ( + lambda optname: self._optdata.get( + optname, {"header": "n"} + ).get("header", "a") + != "n" + ) + + for secname in self._configlist.keys(): + tlines = [] + for optname in self._configlist[secname]: + if mcond(optname): + value = getattr(self, optname) + ttype = self._getTypeStr(optname) + strvalue = ( + ", ".join(map(str, value)) + if ttype.endswith("list") + else str(value) + ) + tlines.append("%s = %s" % (optname, strvalue)) + if len(tlines) > 0: + lines.append("[%s]" % secname) + lines.extend(tlines) + lines.append("") + rv = "\n".join(lines) + "\n" + return rv + + def resetDefault(self, optnames=None): + """Reset all values to their default value. + + :param optnames: list of str, name of options to reset, None for + all options + """ + if optnames == None: + optnames = self._optdata.keys() + for optname in optnames: + if self._optdata.has_key(optname): + setattr(self, optname, self._optdata[optname]["d"]) + self._updateSelf() + return + + ########################################################################### + # IMPORTANT call this method if you want to add options as class attributes!!! + + @classmethod + def initConfigClass(cls): + """Init config class and add options to class. + + IMPORTANT: call this method after you define the metadata of your config + class to add options as class attributes!!! + """ + cls._preInitConfigClass() + + cls.config = ConfigParser.ConfigParser(dict_type=OrderedDict) + cls.args = argparse.ArgumentParser( + description=cls._description, + epilog=cls._epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + cls._configlist = OrderedDict({}) + + cls._optdatalist = cls._optdatalist_default + cls._optdatalist + cls._optdata = dict(cls._optdatalist) + cls._detectAddSectionsC() + for opt in cls._optdatalist: + key = opt[0] + cls._addOptC(key) + + cls._postInitConfigClass() + return + + @classmethod + def _postInitConfigClass(cls): + """Additional processes called after initConfigClass. + + overload it + """ + pass + + @classmethod + def _preInitConfigClass(cls): + """Additional processes called before initConfigClass. + + overload it + """ + pass + + +# VERY IMPORTANT!!! +# add options to class +# initConfigClass(ConfigBase) +# ConfigBase.initConfigClass() + +if __name__ == "__main__": + + test = ConfigBase() + test.updateConfig() diff --git a/src/diffpy/confutils/tools.py b/src/diffpy/confutils/tools.py new file mode 100644 index 0000000..5df9a7c --- /dev/null +++ b/src/diffpy/confutils/tools.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.confutils by DANSE Diffraction group +# Simon J. L. Billinge +# (c) 2013 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.txt for a list of people who contributed. +# See LICENSE.txt for license information. +# +############################################################################## + +import hashlib +import re +import time +import zlib + +import numpy as np + + +def _configPropertyRad(nm): + """Helper function of options delegation, rad to degree.""" + rv = property( + fget=lambda self: np.radians(getattr(self, nm)), + fset=lambda self, val: setattr(self, nm, np.degrees(val)), + fdel=lambda self: delattr(self, nm), + ) + return rv + + +def _configPropertyR(name): + """Create a property that forwards self.name to self.config.name. + + read only + """ + rv = property( + fget=lambda self: getattr(self.config, name), + doc="attribute forwarded to self.config, read-only", + ) + return rv + + +def _configPropertyRW(name): + """Create a property that forwards self.name to self.config.name. + + read and write + """ + rv = property( + fget=lambda self: getattr(self.config, name), + fset=lambda self, value: setattr(self.config, name, value), + fdel=lambda self: delattr(self, name), + doc="attribute forwarded to self.config, read/write", + ) + return rv + + +def str2bool(v): + """Turn string to bool.""" + return v.lower() in ("yes", "true", "t", "1") + + +def opt2Str(opttype, optvalue): + """Turn the value of one option to string, according to the option + type list of values are turned into "value1, value2, value3...". + + :param opttype: string, type of options, for example 'str' or + 'intlist' + :param optvalue: value of the option + :return: string, usually stored in ConfigBase.config + """ + + if opttype.endswith("list"): + rv = ", ".join(map(str, optvalue)) + else: + rv = str(optvalue) + return rv + + +def StrConv(opttype): + """Get the type (a converter function) according to the opttype. + + the function doesn't take list + + :param opttype: string, a type of options, could be 'str', 'int', + 'float', or 'bool' + :return: type (converter function) + """ + if opttype.startswith("str"): + conv = str + elif opttype.startswith("int"): + conv = int + elif opttype.startswith("float"): + conv = float + elif opttype.startswith("bool"): + conv = str2bool + else: + conv = None + return conv + + +def str2Opt(opttype, optvalue): + """Convert the string to value of one option, according to the + option type. + + :param opttype: string, type of options, for example 'str' or + 'intlist' + :param optvalue: string, value of the option + :return: value of the option, usually stored in ConfigBase.config + """ + # base converter + conv = StrConv(opttype) + if opttype.endswith("list"): + temp = re.split(r"\s*,\s*", optvalue) + rv = map(conv, temp) if len(temp) > 0 else [] + else: + rv = conv(optvalue) + return rv + + +class FakeConfigFile(object): + """A fake configfile object used in reading config from header of + data or a real config file.""" + + def __init__(self, configfile, endline="###"): + self.configfile = configfile + self.fp = open(configfile) + self.endline = endline + self.ended = False + self.name = configfile + return + + def readline(self): + """Readline function.""" + line = self.fp.readline() + if line.startswith(self.endline) or self.ended: + rv = "" + self.ended = True + else: + rv = line + return rv + + def close(self): + """Close the file.""" + self.fp.close() + return + + +def checkCRC32(filename): + """Calculate the crc32 value of file. + + :param filename: path to the file + :return: crc32 value of file + """ + try: + fd = open(filename, "rb") + except Exception: + return "Read error" + eachLine = fd.readline() + prev = 0 + while eachLine: + prev = zlib.crc32(eachLine, prev) + eachLine = fd.readline() + fd.close() + return prev + + +def checkMD5(filename, blocksize=65536): + """Calculate the MD5 value of file. + + :param filename: path to the file + :return: md5 value of file + """ + try: + fd = open(filename, "rb") + except Exception: + return "Read error" + buf = fd.read(blocksize) + md5 = hashlib.md5() + while len(buf) > 0: + md5.update(buf) + buf = fd.read(blocksize) + fd.close() + return md5.hexdigest() + + +def checkFileVal(filename): + """Check file integrity using crc32 and md5. It will read file twice + then compare the crc32 and md5. If two results doesn't match, it + will wait until the file is completed written to disk. + + :param filename: path to the file + """ + valflag = False + lastcrc = checkCRC32(filename) + while not valflag: + currcrc = checkCRC32(filename) + if currcrc == lastcrc: + lastmd5 = checkMD5(filename) + time.sleep(0.01) + currmd5 = checkMD5(filename) + if lastmd5 == currmd5: + valflag = True + else: + time.sleep(0.5) + lastcrc = checkCRC32(filename) + return diff --git a/src/diffpy/confutils/version.py b/src/diffpy/confutils/version.py new file mode 100644 index 0000000..7aa6b57 --- /dev/null +++ b/src/diffpy/confutils/version.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.confutils by DANSE Diffraction group +# Simon J. L. Billinge +# (c) 2013 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.txt for a list of people who contributed. +# See LICENSE.txt for license information. +# +############################################################################## +"""Definition of __version__, __date__, __gitsha__.""" + +from configparser import SafeConfigParser + +from pkg_resources import resource_stream + +# obtain version information from the version.cfg file +cp = SafeConfigParser() +cp.readfp(resource_stream(__name__, "version.cfg")) + +__version__ = cp.get("DEFAULT", "version") +__date__ = cp.get("DEFAULT", "date") +__gitsha__ = cp.get("DEFAULT", "commit") + +del cp + +# End of file