diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..4af5eb2 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,14 @@ +coverage: + status: + project: # more options at https://docs.codecov.com/docs/commit-status + default: + target: auto # use the coverage from the base commit, fail if coverage is lower + threshold: 0% # allow the coverage to drop by + +comment: + layout: " diff, flags, files" + behavior: default + require_changes: false + require_base: false # [true :: must have a base report to post] + require_head: false # [true :: must have a head report to post] + hide_project_coverage: false # [true :: only show coverage on the git diff aka patch coverage] diff --git a/.codespell/ignore_lines.txt b/.codespell/ignore_lines.txt new file mode 100644 index 0000000..07fa7c8 --- /dev/null +++ b/.codespell/ignore_lines.txt @@ -0,0 +1,2 @@ +;; Please include filenames and explanations for each ignored line. +;; See https://docs.openverse.org/meta/codespell.html for docs. diff --git a/.codespell/ignore_words.txt b/.codespell/ignore_words.txt new file mode 100644 index 0000000..04b4fcf --- /dev/null +++ b/.codespell/ignore_words.txt @@ -0,0 +1,8 @@ +;; Please include explanations for each ignored word (lowercase). +;; See https://docs.openverse.org/meta/codespell.html for docs. + +;; abbreviation for "materials" often used in a journal title +mater + +;; Frobenius norm used in np.linalg.norm +fro diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..88077af --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +# As of now, flake8 does not natively support configuration via pyproject.toml +# https://github.com/microsoft/vscode-flake8/issues/135 +[flake8] +exclude = + .git, + __pycache__, + build, + dist, + docs/source/conf.py +max-line-length = 79 +# Ignore some style 'errors' produced while formatting by 'black' +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#labels-why-pycodestyle-warnings +extend-ignore = E203 diff --git a/.github/ISSUE_TEMPLATE/bug_feature.md b/.github/ISSUE_TEMPLATE/bug_feature.md new file mode 100644 index 0000000..b3454de --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_feature.md @@ -0,0 +1,16 @@ +--- +name: Bug Report or Feature Request +about: Report a bug or suggest a new feature! +title: "" +labels: "" +assignees: "" +--- + +### Problem + + + +### Proposed solution diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md new file mode 100644 index 0000000..56bcd01 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -0,0 +1,46 @@ +--- +name: Release +about: Checklist and communication channel for PyPI and GitHub release +title: "Ready for PyPI/GitHub release" +labels: "release" +assignees: "" +--- + +### PyPI/GitHub rc-release preparation checklist: + +- [ ] All PRs/issues attached to the release are merged. +- [ ] All the badges on the README are passing. +- [ ] License information is verified as correct. If you are unsure, please comment below. +- [ ] Locally rendered documentation contains all appropriate pages, including API references (check no modules are + missing), tutorials, and other human-written text is up-to-date with any changes in the code. +- [ ] Installation instructions in the README, documentation, and the website are updated. +- [ ] Successfully run any tutorial examples or do functional testing with the latest Python version. +- [ ] Grammar and writing quality are checked (no typos). +- [ ] Install `pip install build twine`, run `python -m build` and `twine check dist/*` to ensure that the package can be built and is correctly formatted for PyPI release. + +Please tag the maintainer (e.g., @username) in the comment here when you are ready for the PyPI/GitHub release. Include any additional comments necessary, such as version information and details about the pre-release here: + +### PyPI/GitHub full-release preparation checklist: + +- [ ] Create a new conda environment and install the rc from PyPI (`pip install ==??`) +- [ ] License information on PyPI is correct. +- [ ] Docs are deployed successfully to `https:///`. +- [ ] Successfully run all tests, tutorial examples or do functional testing. + +Please let the maintainer know that all checks are done and the package is ready for full release. + +### conda-forge release preparation checklist: + + + +- [ ] Ensure that the full release has appeared on PyPI successfully. +- [ ] New package dependencies listed in `conda.txt` and `tests.txt` are added to `meta.yaml` in the feedstock. +- [ ] Close any open issues on the feedstock. Reach out to the maintainer if you have questions. +- [ ] Tag the maintainer for conda-forge release. + +### Post-release checklist + + + +- [ ] Run tutorial examples and conduct functional testing using the installation guide in the README. Attach screenshots/results as comments. +- [ ] Documentation (README, tutorials, API references, and websites) is deployed without broken links or missing figures. diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 0000000..1099d86 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,15 @@ +### What problem does this PR address? + + + +### What should the reviewer(s) do? + + + + diff --git a/.github/workflows/build-wheel-release.yml b/.github/workflows/build-wheel-release.yml new file mode 100644 index 0000000..8409857 --- /dev/null +++ b/.github/workflows/build-wheel-release.yml @@ -0,0 +1,230 @@ +name: Build Wheels and Release + +on: + workflow_dispatch: + push: + tags: + - "*" + +env: + PYTHON_VERSIONS: '["3.11","3.12","3.13"]' + +permissions: + contents: write + actions: read + packages: write + +concurrency: + group: build-wheels-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash {0} + +jobs: + get-python-versions: + runs-on: ubuntu-latest + outputs: + py_ver: ${{ steps.set.outputs.py_ver }} + py_last: ${{ steps.set.outputs.py_last }} + steps: + - id: set + run: | + echo py_ver=$PYTHON_VERSIONS >> $GITHUB_OUTPUT + echo "py_last=${{ fromJson(env.PYTHON_VERSIONS)[0] }}" >> $GITHUB_OUTPUT + + check-tag-on-main: + runs-on: ubuntu-latest + steps: + - name: Checkout repository with full history + uses: actions/checkout@v4 + with: + token: ${{ secrets.PAT_TOKEN }} + fetch-depth: 0 + + - name: Verify tag + run: | + git fetch origin main + TAG_COMMIT=$(git rev-parse ${{ github.ref_name }}) + if git merge-base --is-ancestor "$TAG_COMMIT" origin/main; then + echo "Tag ${{ github.ref_name }} ($TAG_COMMIT) is contained in main" + else + echo "::error::Tag ${{ github.ref_name }} ($TAG_COMMIT) is not found in main. Please release from the main branch." + exit 1 + fi + + check-tag-privilege: + # No third party actions used + uses: scikit-package/release-scripts/.github/workflows/_check-tag-privilege.yml@v0 + with: + maintainer_github_username: sbillinge + + build-sdist: + needs: [check-tag-privilege, get-python-versions, check-tag-on-main] + runs-on: ubuntu-latest + + steps: + - name: Checkout + # GitHub officially-maintained actions + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + # GitHub officially-maintained actions + uses: actions/setup-python@v5 + with: + python-version: ${{ needs.get-python-versions.outputs.py_last }} + + - name: Build docs and remove fonts that blow up the wheel size + run: | + python -m pip install Sphinx sphinx-rtd-theme sphinx-copybutton m2r + cd docs + make html + cd build/html/_static/css/fonts + find . -type f ! -name '*.ttf' -delete + cd ../.. + rm -rf fonts + cd .. + rm -rf _sources + cd .. + rm -rf doctrees + cd ../.. + + - name: Make sdist + run: | + python -m pip install --upgrade pip build cython setuptools setuptools-git-versioning + python -m build --sdist --no-isolation + + - name: Strip source codes + run: | + set -euo pipefail + tar xf dist/diffpy_srxconfutils-*.tar.gz + SRC_DIR=$(find . -maxdepth 1 -type d -name 'diffpy_srxconfutils-*' | head -n1) + find "$SRC_DIR" -type f -name '*.c' -print0 \ + | xargs -0 perl -i.bak -0777 -pe 's{/\*.*?\*/}{}gs' + find "$SRC_DIR" -type f -name '*.c.bak' -delete + tar czf dist/"${SRC_DIR#./}".tar.gz "$SRC_DIR" + rm -rf "$SRC_DIR" + + - name: Upload sdist + # GitHub officially-maintained actions + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/ + retention-days: 1 + + - name: Upload INSTRUCTIONS.txt + uses: actions/upload-artifact@v4 + with: + name: instructions + path: INSTRUCTIONS.txt + + build-wheels: + needs: [build-sdist, get-python-versions] + + name: Build wheels ${{ matrix.python }}-${{ matrix.buildplat }} + runs-on: ${{ matrix.buildplat }} + strategy: + fail-fast: false + matrix: + buildplat: + - ubuntu-latest + - macos-14 + - windows-latest + python: ${{ fromJSON(needs.get-python-versions.outputs.py_ver) }} + steps: + - name: Download sdist + # GitHub officially-maintained actions + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist/ + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Build wheels + run: | + # setuptools-git-versioning only look into .git and PKG-INFO in the top directory + which python; python --version + which pip; pip --version + python -m pip install --upgrade pip build cython setuptools setuptools-git-versioning + tar xf dist/diffpy_srxconfutils-*.tar.gz + cd diffpy_srxconfutils-* + python -m pip wheel . --no-deps --no-build-isolation --wheel-dir ./../dist + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.python }}-${{ matrix.buildplat }} + path: dist/*.whl + + test-wheels: + needs: [build-wheels, get-python-versions] + + name: Test wheels ${{ matrix.python }}-${{ matrix.buildplat }} + runs-on: ${{ matrix.buildplat }} + strategy: + fail-fast: false + matrix: + buildplat: + - ubuntu-latest + - macos-14 + - windows-latest + python: ${{ fromJson(needs.get-python-versions.outputs.py_ver) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download wheels + uses: actions/download-artifact@v4 + with: + name: wheels-${{ matrix.python }}-${{ matrix.buildplat }} + path: dist/ + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install wheels + run: | + python -m pip install --upgrade pip pytest setuptools-git-versioning + python -m pip install dist/*.whl + + - name: Assert no source files in installed package + run: | + # For debugging + # touch $(python -c "import sysconfig, os; print(os.path.join(sysconfig.get_paths()['purelib'], 'diffpy/srxconfutils', 'fake.py'))") + python - << 'EOF' + import os, glob, sys, sysconfig + sp = sysconfig.get_paths()['purelib'] + pkg = os.path.join(sp, 'diffpy', 'srxconfutils') + patterns = [os.path.join(pkg, '*.py'), + os.path.join(pkg, '*.c')] + bad = [] + for p in patterns: + bad.extend(glob.glob(p)) + + if bad: + print("Found leftover source files in installed package:") + for f in bad: + print(" -", f) + sys.exit(1) + + print("No .py or .c files present in diffpy/srxconfutils/") + EOF + + - name: Run tests + run: python -m pytest + + release: + needs: [test-wheels] + uses: ./.github/workflows/release-github.yml + secrets: + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} diff --git a/.github/workflows/check-news-item.yml b/.github/workflows/check-news-item.yml new file mode 100644 index 0000000..4857a2c --- /dev/null +++ b/.github/workflows/check-news-item.yml @@ -0,0 +1,12 @@ +name: Check for News + +on: + pull_request_target: + branches: + - main + +jobs: + check-news-item: + uses: scikit-package/release-scripts/.github/workflows/_check-news-item.yml@v0 + with: + project: diffpy.srxconfutils diff --git a/.github/workflows/matrix-and-codecov-on-merge-to-main.yml b/.github/workflows/matrix-and-codecov-on-merge-to-main.yml new file mode 100644 index 0000000..4745dc9 --- /dev/null +++ b/.github/workflows/matrix-and-codecov-on-merge-to-main.yml @@ -0,0 +1,19 @@ +name: CI + +on: + push: + branches: + - main + release: + types: + - prereleased + - published + workflow_dispatch: + +jobs: + matrix-coverage: + uses: scikit-package/release-scripts/.github/workflows/_matrix-no-codecov-on-merge-to-main.yml@v0 + with: + project: diffpy.srxconfutils + c_extension: false + headless: false diff --git a/.github/workflows/release-github.yml b/.github/workflows/release-github.yml new file mode 100644 index 0000000..3e78d1d --- /dev/null +++ b/.github/workflows/release-github.yml @@ -0,0 +1,163 @@ +name: Release on GitHub + +on: + workflow_call: + secrets: + PAT_TOKEN: + description: "GitHub Personal Access Token" + required: true + +env: + TAG: ${{ github.ref_name }} + +defaults: + run: + shell: bash {0} + +jobs: + prepare-release: + if: ${{ ! contains(github.ref, 'rc') }} + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + ref: main + + - name: Update CHANGELOG + run: | + wget https://raw.githubusercontent.com/scikit-package/release-scripts/v0/.github/workflows/update-changelog.py + python update-changelog.py "$TAG" + rm update-changelog.py + + - name: Commit updated CHANGELOG.rst + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add . + if ! git diff --cached --quiet; then + git commit -m "update changelog for $TAG" + git push origin main + else + echo "No CHANGELOG.rst changes" + fi + + - name: New tag + run: | + git fetch --tags + git tag -d "$TAG" 2>/dev/null || true + git push origin ":$TAG" 2>/dev/null || true + git tag "$TAG" + git push origin "$TAG" + + - name: Get CHANGELOG.txt + run: | + wget https://raw.githubusercontent.com/scikit-package/release-scripts/v0/.github/workflows/get-latest-changelog.py + python get-latest-changelog.py "$TAG" + rm get-latest-changelog.py + + - name: Upload changelog.txt + uses: actions/upload-artifact@v4 + with: + name: changelog + path: CHANGELOG.txt + + release: + needs: [prepare-release] + if: always() + runs-on: ubuntu-latest + env: + REPO_FULL: ${{ github.repository }} + PAT_TOKEN: ${{ secrets.PAT_TOKEN }} + steps: + - name: Check prepare release + run: | + if [[ ${{ needs.prepare-release.result }} != 'success' && "$TAG" != *rc* ]]; then + echo "::error::Skipping release job because prepare-release failed" + exit 78 + fi + echo "Continuing with release job" + + - name: Download built wheels + uses: actions/download-artifact@v4 + with: + path: dist/ + + - name: Download changelog + if: ${{ needs.prepare-release.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: changelog + path: . + + - name: Download instructions + uses: actions/download-artifact@v4 + with: + name: instructions + path: . + + - name: Zip wheels and instructions into dist/srxconfutils-$TAG-wheels.zip + run: | + mkdir -p dist + find dist -type f -name '*.whl' | zip -j dist/srxconfutils-"$TAG"-wheels.zip -@ + zip -j dist/srxconfutils-"$TAG"-wheels.zip INSTRUCTIONS.txt + + - name: Prepare release metadata + id: meta + run: | + if [[ "$TAG" == *rc* ]]; then + PRERELEASE=true + TITLE="Pre-release $TAG" + BODY_RAW="Changelog: https://github.com/$REPO_FULL/commits/$TAG" + else + PRERELEASE=false + TITLE="Release $TAG" + BODY_RAW=$( payload.json + + echo "Release metadata:" + cat payload.json + + - name: Create GitHub Release + id: create_release + run: | + set -euo pipefail + + HTTP_STATUS=$( + curl --silent --output resp.json --write-out "%{http_code}" \ + -X POST "https://api.github.com/repos/$REPO_FULL/releases" \ + -H "Accept: application/vnd.github+json" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $PAT_TOKEN" \ + --data @payload.json + ) + if [[ "$HTTP_STATUS" -ne 201 ]]; then + echo "::error::Failed to create release (status $HTTP_STATUS)" + exit 1 + fi + + UPLOAD_URL=$(jq -r .upload_url resp.json | sed 's/{.*}//') + echo "upload_url=$UPLOAD_URL" >> $GITHUB_OUTPUT + + - name: Upload srxconfutils-$TAG-wheels.zip + if: steps.create_release.outputs.upload_url != '' + run: | + FILE=dist/srxconfutils-$TAG-wheels.zip + echo "Uploading asset: $FILE" + curl --silent --fail --data-binary @"$FILE" \ + -H "Content-Type: application/zip" \ + -H "Authorization: Bearer $PAT_TOKEN" \ + "${{ steps.create_release.outputs.upload_url }}?name=$(basename "$FILE")" diff --git a/.github/workflows/tests-on-pr.yml b/.github/workflows/tests-on-pr.yml new file mode 100644 index 0000000..aa251fc --- /dev/null +++ b/.github/workflows/tests-on-pr.yml @@ -0,0 +1,16 @@ +name: Tests on PR + +on: + pull_request: + workflow_dispatch: + +jobs: + tests-on-pr: + uses: scikit-package/release-scripts/.github/workflows/_tests-on-pr-no-codecov.yml@v0 + with: + project: diffpy.srxconfutils + c_extension: false + headless: false + run: | + conda install pre-commit + pre-commit run --all-files diff --git a/.gitignore b/.gitignore index b9f9b7d..099e294 100644 --- a/.gitignore +++ b/.gitignore @@ -1,44 +1,93 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +*$py.class # C extensions *.so -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -temp -develop-eggs +# Distribution / packaging +.Python +env/ +build/ +_build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +venv/ +*.egg-info/ .installed.cfg -lib -lib64 -tags +*.egg +bin/ +temp/ +tags/ errors.err +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + # Installer logs pip-log.txt +pip-delete-this-directory.txt MANIFEST # Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.tox +.coverage.* +.cache nosetests.xml +coverage.xml +*,cover +.hypothesis/ # Translations *.mo +*.pot # Mr Developer .mr.developer.cfg .project .pydevproject -.settings -# version information -setup.cfg -/dpx/confutils/version.cfg +# Django stuff: +*.log + +# Sphinx documentation +docs/build/ +docs/source/generated/ + +# pytest +.pytest_cache/ + +# PyBuilder +target/ + +# Editor files +# mac +.DS_Store +*~ + +# vim +*.swp +*.swo + +# pycharm +.idea/ + +# VSCode +.vscode/ + +# Ipython Notebook +.ipynb_checkpoints diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..86f162b --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,5 @@ +[settings] +# Keep import statement below line_length character limit +line_length = 79 +multi_line_output = 3 +include_trailing_comma = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0e4a84d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,66 @@ +default_language_version: + python: python3 +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit hooks + autofix_prs: true + autoupdate_branch: "pre-commit-autoupdate" + autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate" + autoupdate_schedule: monthly + skip: [no-commit-to-branch] + submodules: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-added-large-files + - repo: https://github.com/psf/black + rev: 24.4.2 + hooks: + - id: black + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/kynan/nbstripout + rev: 0.7.1 + hooks: + - id: nbstripout + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: no-commit-to-branch + name: Prevent Commit to Main Branch + args: ["--branch", "main"] + stages: [pre-commit] + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + additional_dependencies: + - tomli + # prettier - multi formatter for .json, .yml, and .md files + - repo: https://github.com/pre-commit/mirrors-prettier + rev: f12edd9c7be1c20cfa42420fd0e6df71e42b51ea # frozen: v4.0.0-alpha.8 + hooks: + - id: prettier + additional_dependencies: + - "prettier@^3.2.4" + # docformatter - PEP 257 compliant docstring formatter + - repo: https://github.com/s-weigand/docformatter + rev: 5757c5190d95e5449f102ace83df92e7d3b06c6c + hooks: + - id: docformatter + additional_dependencies: [tomli] + args: [--in-place, --config, ./pyproject.toml] diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..aaa8889 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "latest" + +python: + install: + - requirements: requirements/docs.txt + +sphinx: + configuration: docs/source/conf.py diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..bbc8c0d --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,11 @@ +Authors +======= + + Xiaohao Yang + Billinge Group members + +Contributors +------------ + +For a list of contributors, visit +https://github.com/diffpy/diffpy.srxconfutils/graphs/contributors diff --git a/AUTHORS.txt b/AUTHORS.txt deleted file mode 100644 index 1ea282b..0000000 --- a/AUTHORS.txt +++ /dev/null @@ -1,7 +0,0 @@ -This code is developed by: - - Xiaohao Yang - -This code was developed as part of the xPDFsuite project to create software -and tools for general researchers to use PDF in their work. For more -information on the DiffPy project email sb2896@columbia.edu diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..f29d3b5 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,5 @@ +============= +Release notes +============= + +.. current developments diff --git a/CODE-OF-CONDUCT.rst b/CODE-OF-CONDUCT.rst new file mode 100644 index 0000000..e8199ca --- /dev/null +++ b/CODE-OF-CONDUCT.rst @@ -0,0 +1,133 @@ +===================================== + Contributor Covenant Code of Conduct +===================================== + +Our Pledge +---------- + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socioeconomic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +Our Standards +------------- + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +Enforcement Responsibilities +---------------------------- + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +Scope +----- + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +sb2896@columbia.edu. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +Enforcement Guidelines +---------------------- + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +1. Correction +**************** + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +2. Warning +************* + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +3. Temporary Ban +****************** + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +4. Permanent Ban +****************** + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant `_. + +Community Impact Guidelines were inspired by `Mozilla's code of conduct enforcement ladder `_. + +For answers to common questions about this code of conduct, see the `FAQ `_. `Translations are available `_ diff --git a/LICENSENOTICE.txt b/LICENSENOTICE.rst similarity index 89% rename from LICENSENOTICE.txt rename to LICENSENOTICE.rst index 8fc8b60..db80319 100644 --- a/LICENSENOTICE.txt +++ b/LICENSENOTICE.rst @@ -5,6 +5,6 @@ prohibited. If you don’t know whether or not your anticipated use is under a license, you must contact Prof. Simon Billinge at sb2896@columbia.edu. Use of this software without a license is prohibited. -Copyright 2009-2016, Trustees of Columbia University in the City of New York. +Copyright 2009-2025, Trustees of Columbia University in the City of New York. For more information please email Prof. Simon Billinge at sb2896@columbia.edu diff --git a/MANIFEST.in b/MANIFEST.in index 982d898..f1a78ee 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,12 @@ -recursive-include dpx * -prune doc -exclude MANIFEST.in +graft src +graft tests +graft requirements + +include AUTHORS.rst LICENSE*.rst README.rst + +# Exclude all bytecode files and __pycache__ directories +global-exclude *.py[cod] # Exclude all .pyc, .pyo, and .pyd files. +global-exclude .DS_Store # Exclude Mac filesystem artifacts. +global-exclude __pycache__ # Exclude Python cache directories. +global-exclude .git* # Exclude git files and directories. +global-exclude .idea # Exclude PyCharm project settings. diff --git a/README.rst b/README.rst index 720412f..8a852ff 100644 --- a/README.rst +++ b/README.rst @@ -1,36 +1,103 @@ -dpx.confutils -======================================================================== +|Icon| |title|_ +=============== -Package for processing configurations +.. |title| replace:: diffpy.srxconfutils +.. _title: https://diffpy.github.io/diffpy.srxconfutils +.. |Icon| image:: https://avatars.githubusercontent.com/diffpy + :target: https://diffpy.github.io/diffpy.srxconfutils + :height: 100px -REQUIREMENTS ------------------------------------------------------------------------- +|PythonVersion| |PR| -The dpx.confutils requires Python 2.7 and the following software: +|Black| |Tracking| -* ``numpy`` -* ``traits`` +.. |Black| image:: https://img.shields.io/badge/code_style-black-black + :target: https://github.com/psf/black +.. |PR| image:: https://img.shields.io/badge/PR-Welcome-29ab47ff + :target: https://github.com/diffpy/diffpy.srxconfutils/pulls -INSTALLATION ------------------------------------------------------------------------- +.. |PythonVersion| image:: https://img.shields.io/badge/python-3.11%20|%203.12%20|%203.13-blue -We are going to release conda package for all platform. For general user -please use the installation file and install software. For developor, -you can install dpx.confutils using +.. |Tracking| image:: https://img.shields.io/badge/issue_tracking-github-blue + :target: https://github.com/diffpy/diffpy.srxconfutils/issues - python setup.py install - -Note: the dependency is not specified in the setup.py. You need to install -them yourself. You can use Anaconda or other python enviroment. +Configuration utilities for diffpy project. Part of xPDFsuite. +For more information about the diffpy.srxconfutils library, please consult our `online documentation `_. -CONTACTS ------------------------------------------------------------------------- +Citation +-------- -For more information on diffpy.Structure please visit the project web-page +If you use diffpy.srxconfutils in a scientific publication, we would like you to cite this package as -http://www.diffpy.org/ + Xiaohao Yang, Pavol Juhas, Christopher L. Farrow and Simon J. L. Billinge, xPDFsuite: an end-to-end + software solution for high throughput pair distribution function transformation, visualization and + analysis, arXiv 1402.3163 (2014) -or email Prof. Simon Billinge at sb2896@columbia.edu. \ No newline at end of file +Installation +------------ +``diffpy.srxconfutils`` is normally installed as part of the ``xpdfsuite`` software, so please refer to the +installation instructions detailed in the ``README.rst`` file of ``xpdfsuite`` `here `_. + +Independent Installation +------------------------ +You can also install ``diffpy.srxconfutils`` independently for yourself. + +Assuming you have a wheel file in the current working directory, in an active conda environment please type + + pip install ./diffpy.srxconfutils-VERSION.whl + +where you replace VERSION with the actual version you have so the command matches the filename of the +wheel file you have. + +The commands to create and activate the conda environment with name "conf-env" is + + conda create -n conf-env python=3.13 + conda activate conf-env + +If you don't have conda installed, we recomment you install `miniconda +`_ +To install this software from a Python wheel distribution format execute + + pip install ./diffpy.srxconfutils-VERSION.whl + +If you are a developer, you can also install this package from sources. First, obtain the source archive +from `GitHub `_. +Install the packages in ``./requirements/conda.txt`` and ``./requirements/tests.txt`` +using the `--file`` command: + + conda activate conf-env + conda install --file ./requirements/conda.txt + conda install --file ./requirements/tests.txt + pip install -e . # assuming you are in the top level directory of the package + +After installing the dependencies, ``cd`` into your ``diffpy.srxconfutils`` directory +and run the following :: + + pip install . + +This package also provides command-line utilities. To check the software has been installed correctly, type :: + + diffpy.srxconfutils --version + +You can also type the following command to verify the installation. :: + + python -c "import diffpy.srxconfutils; print(diffpy.srxconfutils.__version__)" + + +To view the basic usage and available commands, type :: + + diffpy.srxconfutils -h + + +Contact +------- + +For more information on diffpy.srxconfutils please visit the project `web-page `_ or email Simon J.L. Billinge group at sb2896@columbia.edu. + +Acknowledgements +---------------- + +``diffpy.srxconfutils`` is built and maintained with `scikit-package `_. diff --git a/cookiecutter.json b/cookiecutter.json new file mode 100644 index 0000000..629bacd --- /dev/null +++ b/cookiecutter.json @@ -0,0 +1,18 @@ +{ + "maintainer_name": "Simon J.L. Billinge group", + "maintainer_email": "sb2896@columbia.edu", + "maintainer_github_username": "sbillinge", + "contributors": "Billinge Group members", + "license_holders": "The Trustees of Columbia University in the City of New York", + "project_name": "diffpy.srxconfutils", + "github_username_or_orgname": "diffpy", + "github_repo_name": "diffpy.srxconfutils", + "conda_pypi_package_dist_name": "diffpy.srxconfutils", + "package_dir_name": "diffpy.srxconfutils", + "project_short_description": "Configuration utilities for project. Part of xPDFsuite", + "project_keywords": "diffpy, pdf, data interpretation", + "minimum_supported_python_version": "3.11", + "maximum_supported_python_version": "3.13", + "project_needs_c_code_compiled": "No", + "project_has_gui_tests": "No" +} diff --git a/dpx/__init__.py b/dpx/__init__.py deleted file mode 100644 index 1988b11..0000000 --- a/dpx/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -""" -Blank namespace package. -""" - -__import__('pkg_resources').declare_namespace(__name__) - -# End of file diff --git a/dpx/confutils/__init__.py b/dpx/confutils/__init__.py deleted file mode 100644 index 8e62eb1..0000000 --- a/dpx/confutils/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -# package version -from dpx.confutils.version import __version__ - -# some convenience imports -from dpx.confutils.config import ConfigBase, ConfigBase - -# unit tests -def test(): - '''Execute all unit tests for the diffpy.pdfgetx package. - Return a unittest TestResult object. - ''' - from dpx.confutils.tests import test - return test() - - -# End of file diff --git a/dpx/confutils/config.py b/dpx/confutils/config.py deleted file mode 100644 index 03f64cc..0000000 --- a/dpx/confutils/config.py +++ /dev/null @@ -1,732 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -''' -package for organizing program configurations. It can read/write configurations -file, parse arguments from command lines, and also parse arguments passed from -method/function calling inside python. - -Note: for python 2.6, argparse and orderedDict is required, install them with -easy_install -''' - - -import ConfigParser -import re -import os -import sys -from functools import partial -import argparse -try: - from collections import OrderedDict -except: - from ordereddict import OrderedDict - -from dpx.confutils.tools import _configPropertyRad, _configPropertyR, \ - _configPropertyRW, str2bool, opt2Str, str2Opt, StrConv, FakeConfigFile - -class ConfigBase(object): - ''' - _optdatalist_default, _optdatalist are metadata used to - initialize the options, see below for examples - - options presents in --help (in cmd), config file, headers have same order as - in these list, so arrange them in right order here. - - optional args to control if the options presents in args, config file or - file header - - 'args' - default is 'a' - if 'a', this option will be available in self.args - if 'n', this option will not be available in self.args - 'config' - default is 'a' - if 'f', this option will present in self.config and be written to - config file only in full mode - if 'a', this option will present in self.config and be written to - config file both in full and short mode - if 'n', this option will not present in self.config - 'header' - default is 'a' - if 'f', this option will be written to header only in full mode - if 'a', this option will be written to header both in full and short - mode - if 'n', this option will not be written to header - - so in short mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - ''' - - # Text to display before the argument help - _description = \ - '''Description of configurations - ''' - # Text to display after the argument help - _epilog = \ - ''' - ''' - - ''' - optdata contains these keys: - these args will be passed to argparse, see the documents of argparse for - detail information - - 'f': full, (positional) - 's': short - 'h': help - 't': type - 'a': action - 'n': nargs - 'd': default - 'c': choices - 'r': required - 'de': dest - 'co': const - ''' - _optdatanamedict = {'h':'help', - 't':'type', - 'a':'action', - 'n':'nargs', - 'd':'default', - 'c':'choices', - 'r':'required', - 'de':'dest', - 'co':'const'} - - # examples, overload it - _optdatalist_default = [ - ['configfile', {'sec':'Control', 'config':'f', 'header':'n', - 's':'c', - 'h':'name of input config file', - 'd':'', }], - ['createconfig', {'sec':'Control', 'config':'n', 'header':'n', - 'h':'create a config file according to default or current values', - 'd':'', }], - ['createconfigfull', {'sec':'Control', 'config':'n', 'header':'n', - 'h':'create a full configurable config file', - 'd':'', }], - ] - # examples, overload it - _optdatalist = [ - ['tifdirectory', {'sec':'Experiment', 'header':'n', - 's':'tifdir', - 'h':'directory of raw tif files', - 'd':'currentdir', }], - ['integrationspace', {'sec':'Experiment', - 'h':'integration space, could be twotheta or qspace', - 'd':'twotheta', - 'c':['twotheta', 'qspace'], }], - ['wavelength', {'sec':'Experiment', - 'h':'wavelength of x-ray, in A', - 'd':0.1000, }], - ['rotationd', {'sec':'Experiment', - 's':'rot', - 'h':'rotation angle of tilt plane, in degree', - 'd':0.0, }], - ['includepattern', {'sec':'Beamline', - 's':'ipattern', - 'h':'file name pattern for included files', - 'n':'*', - 'd':['*.tif'], }], - ['excludepattern', {'sec':'Beamline', - 's':'epattern', - 'h':'file name pattern for excluded files', - 'n':'*', - 'd':['*.dark.tif', '*.raw.tif'], }], - ['fliphorizontal', {'sec':'Beamline', - 'h':'filp the image horizontally', - 'n':'?', - 'co':True, - 'd':False, }], - ['regulartmatrixenable', {'sec':'Others', - 'h':'normalize tmatrix in splitting method', - 'n':'?', - 'co':True, - 'd':False, }], - ['maskedges', {'sec':'Others', 'config':'f', 'header':'f', - 'h':'mask the edge pixels, first four means the number of pixels masked in each edge \ - (left, right, top, bottom), the last one is the radius of a region masked around the corner', - 'n':5, - 'd':[1, 1, 1, 1, 50], }], - ] - - # some default data - # configfile: default config file name - # headertitle: default title of header - _defaultdata = {'configfile': ['config.cfg'], - 'headertitle': 'Configuration information' - } - - - def __init__(self, filename=None, args=None, **kwargs): - ''' - init the class and update the values of options if specified in - filename/args/kwargs - - it will: - 1. call self._preInit method - 2. find the config file if specified in filename/args/kwargs - if failed, try to find default config file - 3. update the options value using filename/args/kwargs - file > args > kwargs - - :param filename: str, file name of the config file - :param args: list of str, args passed from cmd - :param kwargs: dict, optional kwargs - - :return: None - ''' - # call self._preInit - self._preInit(**kwargs) - - # update config, first detect if a default config should be load - filename = self._findDefaultConfigFile(filename, args, **kwargs) - rv = self.updateConfig(filename, args, **kwargs) - return - - # example, overload it - def _preInit(self, **kwargs): - ''' - method called in init process, overload it! - - this method will be called before reading config from file/args/kwargs - ''' - # for name in ['rotation']: - # setattr(self.__class__, name, _configPropertyRad(name+'d')) - # self._configlist['Experiment'].extend(['rotation']) - return - - ########################################################################### - - def _findConfigFile(self, filename=None, args=None, **kwargs): - ''' - find config file, if any config is specified in filename/args/kwargs - then return the filename of config. - - :param filename: str, file name of config file - :param filename: list of str, args passed from cmd - :param kwargs: optional kwargs - - :return: name of config file if found, otherwise None - ''' - rv = None - if (filename != None): - rv = filename - if (args != None): - if ('--configfile' in args) or ('-c' in args): - obj = self.args.parse_args(args) - rv = obj.configfile - if kwargs.has_key('configfile'): - rv = kwargs['configfile'] - return rv - - def _findDefaultConfigFile(self, filename=None, args=None, **kwargs): - ''' - find default config file, if any config is specified in - filename/args/kwargs or in self._defaultdata['configfile'], then return - the filename of config. - - kwargs > args > filename > default - - param filename: str, file name of config file - param filename: list of str, args passed from cmd - param kwargs: optional kwargs - - return: name of config file if found, otherwise None - ''' - rv = self._findConfigFile(filename, args, **kwargs) - if rv == None: - for dconf in self._defaultdata['configfile']: - if (os.path.exists(dconf))and(rv == None): - rv = dconf - return rv - - ########################################################################### - - def _updateSelf(self, optnames=None, **kwargs): - ''' - update the options value, then copy the values in the self.'options' to - self.config - - 1. call self._preUpdateSelf - 2. apply options' value from *self.option* to self.config - 3. call self._postUpdateSelf - - :param optnames: str or list of str, name of options whose value has - been changed, if None, update all options - ''' - # so some check right here - self._preUpdateSelf(**kwargs) - # copy value to self.config - self._copySelftoConfig(optnames) - # so some check right here - self._postUpdateSelf(**kwargs) - return - - # example, overload it - def _preUpdateSelf(self, **kwargs): - ''' - additional process called in self._updateSelf, this method is called - before self._copySelftoConfig(), i.e. before copy options value to - self.config (config file) - ''' - return - - def _postUpdateSelf(self, **kwargs): - ''' - additional process called in self._updateSelf, this method is called - after self._copySelftoConfig(), i.e. before copy options value to - self.config (config file) - ''' - return - - ########################################################################### - - def _getTypeStr(self, optname): - ''' - return the type of option - - :param optname: str, name of option - - :return: string, type of the option - ''' - opttype = self._getTypeStrC(optname) - return opttype - - @classmethod - def _getTypeStrC(cls, optname): - ''' - class method, return the type of option - first try to get type information from metadata, if failed, try - to get type from default value - - :param optname: str, name of option - - :return: string, type of the option - ''' - optdata = cls._optdata[optname] - if optdata.has_key('t'): - opttype = optdata['t'] - else: - value = optdata['d'] - if isinstance(value, str): - opttype = 'str' - elif isinstance(value, bool): - opttype = 'bool' - elif isinstance(value, float): - opttype = 'float' - elif isinstance(value, int): - opttype = 'int' - elif isinstance(value, list): - if len(value) == 0: - opttype = 'strlist' - elif isinstance(value[0], str): - opttype = 'strlist' - elif isinstance(value[0], bool): - opttype = 'boollist' - elif isinstance(value[0], float): - opttype = 'floatlist' - elif isinstance(value[0], int): - opttype = 'intlist' - - return opttype - - ########################################################################### - - def _detectAddSections(self): - ''' - detect sections present in self._optdata and add them to self.config - also add it to self._configlist - ''' - self._detectAddSectionsC(self) - return - - @classmethod - def _detectAddSectionsC(cls): - ''' - class method, detect sections present in self._optdata and add them to self.config - also add it to self._configlist - ''' - # seclist = [self._optdata[key]['sec'] for key in self._optdata.keys()] - seclist = [cls._optdata[opt[0]]['sec'] for opt in cls._optdatalist] - secdict = OrderedDict.fromkeys(seclist) - # for sec in set(seclist): - for sec in secdict.keys(): - cls.config.add_section(sec) - cls._configlist[sec] = [] - return - - def _addOpt(self, optname): - ''' - add options to self.config and self.args and self.*option*, - this will read metadata from self._optdatalist - - :param optname: string, name of option - ''' - self._addOptC(self, optname) - return - - @classmethod - def _addOptC(cls, optname): - ''' - Class method, add options to self.config and self.args and - self.*option*, this will read metadata in self._optdatalist - - :param optname: string, name of option - ''' - optdata = cls._optdata[optname] - opttype = cls._getTypeStrC(optname) - - # replace currentdir in default to os.getcwd() - if optdata['d'] == 'currentdir': - optdata['d'] = os.getcwd() - - # add to cls.'optname' - cls._addOptSelfC(optname, optdata) - - # add to cls.config - secname = optdata['sec'] if optdata.has_key('sec') else 'Others' - cls._configlist[secname].append(optname) - if optdata.get('config', 'a') != 'n': - strvalue = ', '.join(map(str, optdata['d'])) if isinstance(optdata['d'], list) else str(optdata['d']) - cls.config.set(secname, optname, strvalue) - # add to cls.args - if optdata.get('args', 'a') != 'n': - # transform optdata to a dict that can pass to add_argument method - pargs = dict() - for key in optdata.keys(): - if cls._optdatanamedict.has_key(key): - pargs[cls._optdatanamedict[key]] = optdata[key] - pargs['default'] = argparse.SUPPRESS - pargs['type'] = StrConv(opttype) - # add args - if optdata.has_key('f'): - cls.args.add_argument(optname, **pargs) - elif optdata.has_key('s'): - cls.args.add_argument('--' + optname, '-' + optdata['s'], **pargs) - else: - cls.args.add_argument('--' + optname, **pargs) - return - - @classmethod - def _addOptSelfC(cls, optname, optdata): - ''' - class method, assign options value to *self.option*, using metadata - - :param optname: string, name of the option - :param optdata: dict, metadata of the options, get it from self._optdatalist - ''' - setattr(cls, optname, optdata['d']) - return - - def _copyConfigtoSelf(self, optnames=None): - ''' - copy the options' value from self.config to self.*option* - - :param optnames: str or list of str, names of options whose value copied - from self.config to self.*option*'. Set None to update all - ''' - if optnames != None: - optnames = optnames if isinstance(optnames, list) else [optnames] - else: - optnames = [] - for secname in self.config.sections(): - optnames += self.config.options(secname) - - for optname in optnames: - if self._optdata.has_key(optname): - secname = self._optdata[optname]['sec'] - opttype = self._getTypeStr(optname) - optvalue = self.config.get(secname, optname) - setattr(self, optname, str2Opt(opttype, optvalue)) - return - - def _copySelftoConfig(self, optnames=None): - ''' - copy the value from self.*option* to self.config - - :param optname: str or list of str, names of options whose value copied - from self.*option* to self.config. Set None to update all - ''' - if optnames != None: - optnames = optnames if isinstance(optnames, list) else [optnames] - else: - optnames = [] - for secname in self.config.sections(): - optnames += self.config.options(secname) - - for optname in optnames: - if self._optdata.has_key(optname): - secname = self._optdata[optname]['sec'] - opttype = self._getTypeStr(optname) - optvalue = getattr(self, optname) - self.config.set(secname, optname, opt2Str(opttype, optvalue)) - return - - ########################################################################### - - def parseArgs(self, pargs): - ''' - parse args and update the value in self.*option*, this will call the - self.args() to parse args, - - :param pargs: list of string, arguments to parse, usually comming from sys.argv - ''' - obj = self.args.parse_args(pargs) - changedargs = obj.__dict__.keys() - for optname in changedargs: - if self._optdata.has_key(optname): - setattr(self, optname, getattr(obj, optname)) - # update self - if len(changedargs) > 0: - self._updateSelf(changedargs) - return obj - - def parseKwargs(self, **kwargs): - ''' - update self.*option* values according to the kwargs - - :param kwargs: dict, keywords=value - ''' - if kwargs != {}: - changedargs = [] - for optname, optvalue in kwargs.iteritems(): - if self._optdata.has_key(optname): - setattr(self, optname, optvalue) - changedargs.append(optname) - # update self - self._updateSelf(changedargs) - return - - def parseConfigFile(self, filename): - ''' - read a config file and update the self.*option* - - :param filename: str, file name of config file (include path) - ''' - if filename != None: - filename = os.path.abspath(filename) - if os.path.exists(filename): - self.configfile = filename - self._copySelftoConfig() - fileobj = FakeConfigFile(filename) - # self.config.read(filename) - self.config.readfp(fileobj) - self._copyConfigtoSelf() - self._updateSelf() - return - - def updateConfig(self, filename=None, args=None, **kwargs): - ''' - update config according to config file, args(from sys.argv) or **kwargs - - 1. call self._preUpdateConfig() - 2. process file/args/kwargs passed to this method, - 3. read a configfile if specified in args or kwargs - 4. call self._postUpdateConfig() - 5. write config file if specified in args/kwargs - - :param filename: str, file name of the config file - :param args: list of str, args passed from cmd, - :param kwargs: dict, optional kwargs - - :return: True if anything updated, False if nothing updated - ''' - # call self._preUpdateConfig - self._preUpdateConfig(**kwargs) - - filename = self._findConfigFile(filename, args, **kwargs) - if filename != None: - rv = self.parseConfigFile(filename) - if args != None: - rv = self.parseArgs(args) - if kwargs != {}: - rv = self.parseKwargs(**kwargs) - - if (filename == None)and((args == None)or(args == []))and(kwargs == {}): - rv = self._updateSelf() - - # call self._callbackUpdateConfig - self._postUpdateConfig(**kwargs) - - # write config file - self._createConfigFile() - return rv - - def _preUpdateConfig(self, **kwargs): - ''' - Method called before parsing args or kwargs or config file, in self.updateConfig - ''' - return - - def _postUpdateConfig(self, **kwargs): - ''' - Method called after parsing args or kwargs or config file, in self.updateConfig - ''' - return - - ########################################################################### - def _createConfigFile(self): - ''' - write output config file if specfied in configuration - the filename is specified by self.createconfig - ''' - if (self.createconfig != '')and(self.createconfig != None): - self.writeConfig(self.createconfig, 'short') - self.createconfig = '' - if (self.createconfigfull != '')and(self.createconfigfull != None): - self.writeConfig(self.createconfigfull, 'full') - self.createconfigfull = '' - return - - def writeConfig(self, filename, mode='short', changeconfigfile=True): - ''' - write config to file. the file is compatiable with python package ConfigParser - - :param filename: string, name of file - :param mode: string, 'short' or 'full' ('s' or 'f'). - in short mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - ''' - if changeconfigfile: - self.configfile = os.path.abspath(filename) - self._updateSelf() - # func decide if wirte the option to config according to mode - # options not present in self._optdata will not be written to config - if mode.startswith('s'): - mcond = lambda optname: self._optdata.get(optname, {'config':'n'}).get('config', 'a') == 'a' - else: - mcond = lambda optname: self._optdata.get(optname, {'config':'n'}).get('config', 'a') != 'n' - - lines = [] - for section in self.config._sections: - tlines = [] - for (key, value) in self.config._sections[section].items(): - if (key != "__name__") and mcond(key): - tlines.append("%s = %s" % (key, str(value).replace('\n', '\n\t'))) - if len(tlines) > 0: - lines.append("[%s]" % section) - lines.extend(tlines) - lines.append('') - rv = "\n".join(lines) + "\n" - fp = open(filename, 'w') - fp.write(rv) - fp.close() - return - - def getHeader(self, title=None, mode='full'): - ''' - get a header of configurations values, - - :param title: str, title of header, if None, try to get it from self.defaultvalue - :param mode: string, 'short' or 'full' ('s' or 'f'). - in short mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - - :return: string, lines with line break that can be directly writen to a text file - ''' - - lines = [] - title = '# %s #' % (self._defaultdata['headertitle'] if title == None else title) - lines.append(title) - # func decide if wirte the option to header according to mode - # options not present in self._optdata will not be written to header - if mode.startswith('s'): - mcond = lambda optname: self._optdata.get(optname, {'header':'n'}).get('header', 'a') == 'a' - else: - mcond = lambda optname: self._optdata.get(optname, {'header':'n'}).get('header', 'a') != 'n' - - for secname in self._configlist.keys(): - tlines = [] - for optname in self._configlist[secname]: - if mcond(optname): - value = getattr(self, optname) - ttype = self._getTypeStr(optname) - strvalue = ', '.join(map(str, value)) if ttype.endswith('list') else str(value) - tlines.append("%s = %s" % (optname, strvalue)) - if len(tlines) > 0: - lines.append("[%s]" % secname) - lines.extend(tlines) - lines.append('') - rv = "\n".join(lines) + "\n" - return rv - - def resetDefault(self, optnames=None): - ''' - reset all values to their default value - - :param optnames: list of str, name of options to reset, None for all options - ''' - if optnames == None: - optnames = self._optdata.keys() - for optname in optnames: - if self._optdata.has_key(optname): - setattr(self, optname, self._optdata[optname]['d']) - self._updateSelf() - return - - ########################################################################### - #IMPORTANT call this method if you want to add options as class attributes!!! - - @classmethod - def initConfigClass(cls): - ''' - init config class and add options to class - - IMPORTANT call this method after you define the metadata of your config - class to add options as class attributes!!! - ''' - cls._preInitConfigClass() - - cls.config = ConfigParser.ConfigParser(dict_type=OrderedDict) - cls.args = argparse.ArgumentParser(description=cls._description, - epilog=cls._epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) - cls._configlist = OrderedDict({}) - - cls._optdatalist = cls._optdatalist_default + cls._optdatalist - cls._optdata = dict(cls._optdatalist) - cls._detectAddSectionsC() - for opt in cls._optdatalist: - key = opt[0] - cls._addOptC(key) - - cls._postInitConfigClass() - return - - @classmethod - def _postInitConfigClass(cls): - ''' - additional processes called after initConfigClass - - overload it - ''' - pass - - @classmethod - def _preInitConfigClass(cls): - ''' - additional processes called before initConfigClass - - overload it - ''' - pass - -#VERY IMPORTANT!!! -# add options to class -# initConfigClass(ConfigBase) -# ConfigBase.initConfigClass() - -if __name__ == '__main__': - - test = ConfigBase() - test.updateConfig() diff --git a/dpx/confutils/configtraits.py b/dpx/confutils/configtraits.py deleted file mode 100644 index a4dda2a..0000000 --- a/dpx/confutils/configtraits.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -''' -package for organizing program configurations. It can read/write configurations -file, parse arguments from command lines, and also parse arguments passed from -method/function calling inside python. - -This one is similar to ConfigBase but use Traits, so every option (self.*option* is a trait) - -Note: for python 2.6, argparse and orderedDict is required, install them with easy_install -''' - -import ConfigParser -import re -import os -import sys -from functools import partial -import argparse - -from traits.api import Directory, String, List, Enum, Bool, File, Float, Int, \ - HasTraits, Property, Range, cached_property, Str, Instance, Array,\ - Event, CFloat, CInt, on_trait_change -from traitsui.api import Item, Group, View - -from dpx.confutils.tools import _configPropertyRad, _configPropertyR, _configPropertyRW, \ - str2bool, opt2Str, str2Opt, StrConv -from dpx.confutils.config import ConfigBase - -class ConfigBaseTraits(HasTraits, ConfigBase): - ''' - _optdatalist_default, _optdatalist are metadata used to - initialize the options, see below for examples - - options presents in --help (in cmd), config file, headers have same order as - in these list, so arrange them in right order here. - - optional args to control if the options presents in args, config file or - file header - - 'args' - default is 'a' - if 'a', this option will be available in self.args - if 'n', this option will not be available in self.args - 'config' - default is 'a' - if 'f', this option will present in self.config and be written to - config file only in full mode - if 'a', this option will present in self.config and be written to - config file both in full and short mode - if 'n', this option will not present in self.config - 'header' - default is 'a' - if 'f', this option will be written to header only in full mode - if 'a', this option will be written to header both in full and short - mode - if 'n', this option will not be written to header - - so in short mode, all options with 'a' will be written, in full mode, - all options with 'a' or 'f' will be written - ''' - - # Text to display before the argument help - _description = \ - '''Description of configurations - ''' - # Text to display after the argument help - _epilog = \ - ''' - ''' - - ''' - optdata contains these keys: - these args will be passed to argparse, see the documents of argparse for - detail information - - 'f': full, (positional) - 's': short - 'h': help - 't': type - 'a': action - 'n': nargs - 'd': default - 'c': choices - 'r': required - 'de': dest - 'co': const - - additional options for traits: - 'tt': traits type - 'l': traits label - ''' - _optdatanamedict = {'h':'help', - 't':'type', - 'a':'action', - 'n':'nargs', - 'd':'default', - 'c':'choices', - 'r':'required', - 'de':'dest', - 'co':'const'} - _traitstypedict = { - 'str': String, - 'int': CInt, - 'float': CFloat, - 'bool': Bool, - 'file': File, - 'directory': Directory, - 'strlist':List, - 'intlist':List, - 'floatlist':List, - 'boollist':List, - 'array':Array, - } - - #examples, overload it - _optdatalist_default = [ - ['configfile',{'sec':'Control', 'config':'f', 'header':'n', - 'l':'Config File', - 'tt':'file', - 's':'c', - 'h':'name of input config file', - 'd':'',}], - ['createconfig',{'sec':'Control', 'config':'n', 'header':'n', - 'h':'create a config file according to default or current values', - 'd':'',}], - ['createconfigfull',{'sec':'Control', 'config':'n', 'header':'n', - 'h':'create a full configurable config file', - 'd':'',}], - ] - #examples, overload it - _optdatalist = [ - ['tifdirectory',{'sec':'Experiment', 'header':'n', - 'tt':'directory', - 'l':'Tif directory', - 's':'tifdir', - 'h':'directory of raw tif files', - 'd':'currentdir',}], - ['integrationspace',{'sec':'Experiment', - 'l':'Integration space', - 'h':'integration space, could be twotheta or qspace', - 'd':'twotheta', - 'c':['twotheta','qspace'],}], - ['wavelength',{'sec':'Experiment', - 'l':'Wavelength', - 'h':'wavelength of x-ray, in A', - 'd':0.1000,}], - ['rotationd',{'sec':'Experiment', - 'l':'Tilt Rotation', - 's':'rot', - 'h':'rotation angle of tilt plane, in degree', - 'd':0.0,}], - ['includepattern',{'sec':'Beamline','header':'n','config':'f', - 'l':'Include', - 's':'ipattern', - 'h':'file name pattern for included files', - 'n':'*', - 'd':['*.tif'],}], - ['excludepattern',{'sec':'Beamline','header':'n','config':'f', - 'l':'Exclude', - 's':'epattern', - 'h':'file name pattern for excluded files', - 'n':'*', - 'd':['*.dark.tif', '*.raw.tif'],}], - ['fliphorizontal',{'sec':'Beamline','header':'n','config':'f', - 'l':'Filp horizontally', - 'h':'filp the image horizontally', - 'n':'?', - 'co':True, - 'd':False,}], - ['maskedges',{'sec':'Others','config':'f', - 'tt':'array', - 'l':'Mask edges', - 'h':'mask the edge pixels, first four means the number of pixels masked in each edge \ - (left, right, top, bottom), the last one is the radius of a region masked around the corner', - 'n':5, - 'd':[10,10,10,10,100],}], - ] - - #default config file path and name - _defaultdata = {'configfile': ['config.cfg'], - 'headertitle': 'Configuration information' - } - - def __init__(self, filename=None, args=None, **kwargs): - ''' - init the class and update the values of options if specified in - filename/args/kwargs - - it will: - 1. init class using HasTraits - 2. call self._preInit method - 3. find the config file if specified in filename/args/kwargs - if failed, try to find default config file - 4. update the options value using filename/args/kwargs - file > args > kwargs - 5. call self._postInitTraits() - - :param filename: str, file name of the config file - :param args: list of str, args passed from cmd - :param kwargs: dict, optional kwargs - - :return: None - ''' - HasTraits.__init__(self) - ConfigBase.__init__(self, filename, args, **kwargs) - - self._postInitTraits() - return - - def _postInitTraits(self): - ''' - additional init process called after traits init - ''' - return - - @classmethod - def _addOptSelfC(cls, optname, optdata): - ''' - class method, assign options value to *self.option*, using metadata, - this one will create traits objects for each option - - :param optname: string, name of the option - :param optdata: dict, metadata of the options, get it from self._optdatalist - ''' - #value type - vtype = cls._getTypeStrC(optname) - ttype = optdata.get('tt', vtype) - ttype = cls._traitstypedict[ttype] - kwargs = {'label':optdata['l'] if optdata.has_key('l') else optname, - 'desc':optdata['h'], - } - args = [optdata['d']] - if optdata.has_key('c'): - ttype = Enum - args = [optdata['c']] - kwargs['value']=optdata['d'] - if ttype == Array: - args = [] - kwargs['value']=optdata['d'] - obj = ttype(*args, **kwargs) - cls.add_class_trait(optname, obj) - return - -#ConfigBaseTraits.initConfigClass() - -if __name__=='__main__': - test = ConfigBaseTraits(filename='temp.cfg') - test.updateConfig() - test.configure_traits() diff --git a/dpx/confutils/tools.py b/dpx/confutils/tools.py deleted file mode 100644 index 2ad874a..0000000 --- a/dpx/confutils/tools.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -import numpy as np -import re -import time -import zlib -import hashlib -from pkgutil import iter_modules - -def module_exists(module_name): - return module_name in [tuple_[1] for tuple_ in iter_modules()] - -def module_exists_lower(module_name): - return module_name.lower() in [tuple_[1].lower() for tuple_ in iter_modules()] - -def _configPropertyRad(nm): - ''' - helper function of options delegation, rad to degree - ''' - rv = property(fget=lambda self: np.radians(getattr(self, nm)), - fset=lambda self, val: setattr(self, nm, np.degrees(val)), - fdel=lambda self: delattr(self, nm)) - return rv - -def _configPropertyR(name): - ''' - Create a property that forwards self.name to self.config.name. - - read only - ''' - rv = property(fget=lambda self: getattr(self.config, name), - doc='attribute forwarded to self.config, read-only') - return rv - -def _configPropertyRW(name): - ''' - Create a property that forwards self.name to self.config.name. - - read and write - ''' - rv = property(fget=lambda self: getattr(self.config, nm), - fset=lambda self, value: setattr(self.config, nm, value), - fdel=lambda self: delattr(self, nm), - doc='attribute forwarded to self.config, read/write') - return rv - -def str2bool(v): - ''' - turn string to bool - ''' - return v.lower() in ("yes", "true", "t", "1") - -def opt2Str(opttype, optvalue): - ''' - turn the value of one option to string, according to the option type - list of values are truned into "value1, value2, value3..." - - :param opttype: string, type of opitons, for example 'str' or 'intlist' - :param optvalue: value of the option - - :return: string, usually stored in ConfigBase.config - ''' - - if opttype.endswith('list'): - rv = ', '.join(map(str, optvalue)) - else: - rv = str(optvalue) - return rv - -def StrConv(opttype): - ''' - get the type (or converter function) according to the opttype - - the function doesn't take list - ''' - if opttype.startswith('str'): - conv = str - elif opttype.startswith('int'): - conv = int - elif opttype.startswith('float'): - conv = float - elif opttype.startswith('bool'): - conv = str2bool - else: - conv = None - return conv - -def str2Opt(opttype, optvalue): - ''' - convert the string to value of one option, according to the option type - - :param opttype: string, type of opitons, for example 'str' or 'intlist' - :param optvalue: string, value of the option - - :return: value of the option, usually stored in ConfigBase.config - ''' - # base converter - conv = StrConv(opttype) - if opttype.endswith('list'): - temp = re.split('\s*,\s*', optvalue) - rv = map(conv, temp) if len(temp) > 0 else [] - else: - rv = conv(optvalue) - return rv - -class FakeConfigFile(object): - ''' - A fake configfile object used in reading config from header of data - or a real config file. - ''' - def __init__(self, configfile, endline='###'): - self.configfile = configfile - self.fp = open(configfile) - self.endline = endline - self.ended = False - self.name = configfile - return - - def readline(self): - ''' - readline function - ''' - line = self.fp.readline() - if line.startswith(self.endline) or self.ended: - rv = '' - self.ended = True - else: - rv = line - return rv - - def close(self): - ''' - close the file - ''' - self.fp.close() - return - -def checkCRC32(filename): - ''' - calculate the crc32 value of file - - :param filename: path to the file - - :return: crc32 value of file - ''' - try: - fd = open(filename, 'rb') - except: - return 'Read error' - eachLine = fd.readline() - prev = 0 - while eachLine: - prev = zlib.crc32(eachLine, prev) - eachLine = fd.readline() - fd.close() - return prev - -def checkMD5(filename, blocksize=65536): - ''' - calculate the MD5 value of file - - :param filename: path to the file - - :return: md5 value of file - ''' - try: - fd = open(filename, 'rb') - except: - return 'Read error' - buf = fd.read(blocksize) - md5 = hashlib.md5() - while len(buf) > 0: - md5.update(buf) - buf = fd.read(blocksize) - fd.close() - return md5.hexdigest() - -def checkFileVal(filename): - ''' - check file integrity using crc32 and md5. It will read file twice then - compare the crc32 and md5. If two results doesn't match, it will wait until - the file is completed written to disk. - - :param filename: path to the file - ''' - valflag = False - lastcrc = checkCRC32(filename) - while not valflag: - currcrc = checkCRC32(filename) - if currcrc == lastcrc: - lastmd5 = checkMD5(filename) - time.sleep(0.01) - currmd5 = checkMD5(filename) - if lastmd5 == currmd5: - valflag = True - else: - time.sleep(0.5) - lastcrc = checkCRC32(filename) - return diff --git a/dpx/confutils/version.py b/dpx/confutils/version.py deleted file mode 100644 index 4f60568..0000000 --- a/dpx/confutils/version.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# -# dpx.confutils by Simon J. L. Billinge group -# (c) 2013 Trustees of the Columbia University -# in the City of New York. All rights reserved. -# -# File coded by: Xiaohao Yang -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSENOTICE.txt for license information. -# -############################################################################## - -"""Definition of __version__ and __date__ for this package. -""" - -# obtain version information -from pkg_resources import get_distribution -_pkgname = __name__.rsplit('.', 1)[0] -__version__ = get_distribution(_pkgname).version - -# we assume that tag_date was used and __version__ ends in YYYYMMDD -__date__ = __version__[-8:-4] + '-' + \ - __version__[-4:-2] + '-' + __version__[-2:] - -# End of file diff --git a/news/TEMPLATE.rst b/news/TEMPLATE.rst new file mode 100644 index 0000000..790d30b --- /dev/null +++ b/news/TEMPLATE.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* diff --git a/news/skpkg.rst b/news/skpkg.rst new file mode 100644 index 0000000..b4d8767 --- /dev/null +++ b/news/skpkg.rst @@ -0,0 +1,25 @@ +**Added:** + +* + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* Support scikit-package Level 5 standard (https://scikit-package.github.io/scikit-package/). + +* Port legacy Python 2 code to support Python 3. + +**Security:** + +* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..80824ab --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,88 @@ +[build-system] +requires = ["setuptools>=62.0", "setuptools-git-versioning>=2.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "diffpy.srxconfutils" +dynamic=['version', 'dependencies'] +authors = [ + { name="Simon J.L. Billinge group", email="sb2896@columbia.edu" }, +] +maintainers = [ + { name="Simon J.L. Billinge group", email="sb2896@columbia.edu" }, +] +description = "Configuration utilities for dpx project. Part of xPDFsuite" +keywords = ['diffpy', 'pdf', 'data interpretation'] +readme = "README.rst" +requires-python = ">=3.11, <3.14" +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: Free To Use But Restricted', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Topic :: Scientific/Engineering :: Physics', + 'Topic :: Scientific/Engineering :: Chemistry', +] + +[project.urls] +Homepage = "https://github.com/diffpy/diffpy.srxconfutils/" +Issues = "https://github.com/diffpy/diffpy.srxconfutils/issues/" + +[tool.setuptools-git-versioning] +enabled = true +template = "{tag}" +dev_template = "{tag}" +dirty_template = "{tag}" + +[tool.setuptools.packages.find] +where = ["src"] # list of folders that contain the packages (["."] by default) +include = ["*"] # package names should match these glob patterns (["*"] by default) +exclude = [] # exclude packages matching these glob patterns (empty by default) +namespaces = false # to disable scanning PEP 420 namespaces (true by default) + +[project.scripts] +diffpy-srxconfutils = "diffpy.srxconfutils.app:main" + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements/pip.txt"]} + +[tool.codespell] +exclude-file = ".codespell/ignore_lines.txt" +ignore-words = ".codespell/ignore_words.txt" +skip = "*.cif,*.dat,*agr" + +[tool.docformatter] +recursive = true +wrap-summaries = 72 +wrap-descriptions = 72 + +[tool.black] +line-length = 79 +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | \.rst + | \.txt + | _build + | buck-out + | build + | dist + + # The following are specific to Black, you probably don't want those. + | blib2to3 + | tests/data +)/ +''' diff --git a/requirements/conda.txt b/requirements/conda.txt new file mode 100644 index 0000000..73b057e --- /dev/null +++ b/requirements/conda.txt @@ -0,0 +1,3 @@ +numpy +traits +traitsui diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 0000000..5f34c6e --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1,5 @@ +sphinx +sphinx_rtd_theme +sphinx-copybutton +doctr +m2r diff --git a/requirements/pip.txt b/requirements/pip.txt new file mode 100644 index 0000000..73b057e --- /dev/null +++ b/requirements/pip.txt @@ -0,0 +1,3 @@ +numpy +traits +traitsui diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 0000000..a727786 --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,6 @@ +flake8 +pytest +codecov +coverage +pytest-cov +pytest-env diff --git a/setup.py b/setup.py deleted file mode 100755 index 57f2e07..0000000 --- a/setup.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python - -# Installation script for dpx.confutils - -"""dpx.confutils - configuration utilities for dpx project - -Packages: dpx.confutils -Scripts: None -""" - -import os -from setuptools import setup, find_packages -from setuptools.command.build_py import build_py -import py_compile - - -class custom_build_pyc(build_py): - - def byte_compile(self, files): - for file in files: - if file.endswith('.py'): - py_compile.compile(file) - os.unlink(file) - -# Use this version when git data are not available, like in git zip archive. -# Update when tagging a new release. -FALLBACK_VERSION = '1.0' - -# versioncfgfile holds version data for git commit hash and date. -# It must reside in the same directory as version.py. -MYDIR = os.path.dirname(os.path.abspath(__file__)) -versioncfgfile = os.path.join(MYDIR, 'dpx', 'confutils', 'version.cfg') -gitarchivecfgfile = versioncfgfile.replace('version.cfg', 'gitarchive.cfg') - - -def gitinfo(): - from subprocess import Popen, PIPE - kw = dict(stdout=PIPE, cwd=MYDIR) - proc = Popen(['git', 'describe', '--match=v[[:digit:]]*'], **kw) - desc = proc.stdout.read() - proc = Popen(['git', 'log', '-1', '--format=%H %at %ai'], **kw) - glog = proc.stdout.read() - rv = {} - rv['version'] = '.post'.join(desc.strip().split('-')[:2]).lstrip('v') - rv['commit'], rv['timestamp'], rv['date'] = glog.strip().split(None, 2) - return rv - - -def getversioncfg(): - import re - from ConfigParser import RawConfigParser - vd0 = dict(version=FALLBACK_VERSION, commit='', date='', timestamp=0) - # first fetch data from gitarchivecfgfile, ignore if it is unexpanded - g = vd0.copy() - cp0 = RawConfigParser(vd0) - cp0.read(gitarchivecfgfile) - if '$Format:' not in cp0.get('DEFAULT', 'commit'): - g = cp0.defaults() - mx = re.search(r'\btag: v(\d[^,]*)', g.pop('refnames')) - if mx: - g['version'] = mx.group(1) - # then try to obtain version data from git. - gitdir = os.path.join(MYDIR, '.git') - if os.path.exists(gitdir) or 'GIT_DIR' in os.environ: - try: - g = gitinfo() - except OSError: - pass - # finally, check and update the active version file - cp = RawConfigParser() - cp.read(versioncfgfile) - d = cp.defaults() - rewrite = not d or (g['commit'] and ( - g['version'] != d.get('version') or g['commit'] != d.get('commit'))) - if rewrite: - cp.set('DEFAULT', 'version', g['version']) - cp.set('DEFAULT', 'commit', g['commit']) - cp.set('DEFAULT', 'date', g['date']) - cp.set('DEFAULT', 'timestamp', g['timestamp']) - cp.write(open(versioncfgfile, 'w')) - return cp - -versiondata = getversioncfg() - - -def dirglob(d, *patterns): - from glob import glob - rv = [] - for p in patterns: - rv += glob(os.path.join(d, p)) - return rv - -# define distribution -setup_args = dict( - name='dpx.confutils', - cmdclass=dict(build_py=custom_build_pyc), - version=versiondata.get('DEFAULT', 'version'), - namespace_packages=['dpx'], - packages=find_packages(), - include_package_data=True, - zip_safe=False, - entry_points={ - # define console_scripts here, see setuptools docs for details. - }, - - author='Simon J.L. Billinge', - author_email='sb2896@columbia.edu', - description='configuration utilities for dpx project', - maintainer='Xiaohao Yang', - maintainer_email='sodestiny1@gmail.com', - license='see LICENSENOTICE.txt', - url='', - keywords='dpx configuration utilities', - classifiers=[ - # List of possible values at - # http://pypi.python.org/pypi?:action=list_classifiers - 'Development Status :: 5 - Production/Stable', - 'Environment :: MacOS X', - 'Environment :: Win32 (MS Windows)', - 'Environment :: X11 Applications', - 'Intended Audience :: Science/Research', - 'Operating System :: MacOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Topic :: Scientific/Engineering :: Physics', - ], -) - -if __name__ == '__main__': - setup(**setup_args) - -# End of file diff --git a/src/diffpy/__init__.py b/src/diffpy/__init__.py new file mode 100644 index 0000000..2af95ff --- /dev/null +++ b/src/diffpy/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2013-2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Xiaohao Yang and Billinge Group members. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.srxconfutils/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## diff --git a/src/diffpy/srxconfutils/__init__.py b/src/diffpy/srxconfutils/__init__.py new file mode 100644 index 0000000..e07b832 --- /dev/null +++ b/src/diffpy/srxconfutils/__init__.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2013-2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Xiaohao Yang and Billinge Group members. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.srxconfutils/graphs/contributors +# +# See LICENSE.rst for license information. +# +############################################################################## +"""Configuration utilities for project. + +Part of xPDFsuite +""" + +# package version +from diffpy.srxconfutils.version import __version__ # noqa + +# silence the pyflakes syntax checker +assert __version__ or True + +# End of file diff --git a/src/diffpy/srxconfutils/config.py b/src/diffpy/srxconfutils/config.py new file mode 100644 index 0000000..87970ad --- /dev/null +++ b/src/diffpy/srxconfutils/config.py @@ -0,0 +1,820 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.srxconfutils by Simon J. L. Billinge group +# (c) 2013-2025 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.rst for a list of people who contributed. +# See LICENSENOTICE.rst for license information. +# +############################################################################## +"""Package for organizing program configurations. It can read/write +configurations file, parse arguments from command lines, and also parse +arguments passed from method/function calling inside python. + +Note: for python 2.6, argparse and orderedDict is required, install them with +easy_install +""" + + +import argparse +import os +from configparser import ConfigParser + +try: + from collections import OrderedDict +except Exception: + from ordereddict import OrderedDict + +from diffpy.srxconfutils.tools import FakeConfigFile, StrConv, opt2Str, str2Opt + + +class ConfigBase(object): + """_optdatalist_default, _optdatalist are metadata used to + initialize the options, see below for examples. + + options presents in --help (in cmd), config file, headers have same order + as in these list, so arrange them in right order here. + + optional args to control if the options presents in args, config file or + file header + + 'args' - default is 'a' + if 'a', this option will be available in self.args + if 'n', this option will not be available in self.args + 'config' - default is 'a' + if 'f', this option will present in self.config and be written to + config file only in full mode + if 'a', this option will present in self.config and be written to + config file both in full and short mode + if 'n', this option will not present in self.config + 'header' - default is 'a' + if 'f', this option will be written to header only in full mode + if 'a', this option will be written to header both in full and short + mode + if 'n', this option will not be written to header + + so in short mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + """ + + # Text to display before the argument help + _description = """Description of configurations + """ + # Text to display after the argument help + _epilog = """ + """ + + """ + optdata contains these keys: + these args will be passed to argparse, see the documents of argparse for + detail information + + 'f': full, (positional) + 's': short + 'h': help + 't': type + 'a': action + 'n': nargs + 'd': default + 'c': choices + 'r': required + 'de': dest + 'co': const + """ + _optdatanamedict = { + "h": "help", + "t": "type", + "a": "action", + "n": "nargs", + "d": "default", + "c": "choices", + "r": "required", + "de": "dest", + "co": "const", + } + + # examples, overload it + _optdatalist_default = [ + [ + "configfile", + { + "sec": "Control", + "config": "f", + "header": "n", + "s": "c", + "h": "name of input config file", + "d": "", + }, + ], + [ + "createconfig", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": ( + "create a config file according to " + "default or current values" + ), + "d": "", + }, + ], + [ + "createconfigfull", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": "create a full configurable config file", + "d": "", + }, + ], + ] + # examples, overload it + _optdatalist = [ + [ + "tifdirectory", + { + "sec": "Experiment", + "header": "n", + "s": "tifdir", + "h": "directory of raw tif files", + "d": "currentdir", + }, + ], + [ + "integrationspace", + { + "sec": "Experiment", + "h": "integration space, could be twotheta or qspace", + "d": "twotheta", + "c": ["twotheta", "qspace"], + }, + ], + [ + "wavelength", + { + "sec": "Experiment", + "h": "wavelength of x-ray, in A", + "d": 0.1000, + }, + ], + [ + "rotationd", + { + "sec": "Experiment", + "s": "rot", + "h": "rotation angle of tilt plane, in degree", + "d": 0.0, + }, + ], + [ + "includepattern", + { + "sec": "Beamline", + "s": "ipattern", + "h": "file name pattern for included files", + "n": "*", + "d": ["*.tif"], + }, + ], + [ + "excludepattern", + { + "sec": "Beamline", + "s": "epattern", + "h": "file name pattern for excluded files", + "n": "*", + "d": ["*.dark.tif", "*.raw.tif"], + }, + ], + [ + "fliphorizontal", + { + "sec": "Beamline", + "h": "flip the image horizontally", + "n": "?", + "co": True, + "d": False, + }, + ], + [ + "regulartmatrixenable", + { + "sec": "Others", + "h": "normalize tmatrix in splitting method", + "n": "?", + "co": True, + "d": False, + }, + ], + [ + "maskedges", + { + "sec": "Others", + "config": "f", + "header": "f", + "h": ( + "mask the edge pixels, first four means the " + "number of pixels masked in each edge " + "(left, right, top, bottom), the last one is " + "the radius of a region masked around the corner" + ), + "n": 5, + "d": [1, 1, 1, 1, 50], + }, + ], + ] + + # some default data + # configfile: default config file name + # headertitle: default title of header + _defaultdata = { + "configfile": ["config.cfg"], + "headertitle": "Configuration information", + } + + def __init__(self, filename=None, args=None, **kwargs): + """Init the class and update the values of options if specified + in filename/args/kwargs. + + it will: + 1. call self._preInit method + 2. find the config file if specified in filename/args/kwargs + if failed, try to find default config file + 3. update the options value using filename/args/kwargs + file > args > kwargs + + :param filename: str, file name of the config file + :param args: list of str, args passed from cmd + :param kwargs: dict, optional kwargs + + :return: None + """ + # call self._preInit + self._preInit(**kwargs) + + # update config, first detect if a default config should be load + filename = self._findDefaultConfigFile(filename, args, **kwargs) + self.updateConfig(filename, args, **kwargs) + return + + # example, overload it + def _preInit(self, **kwargs): + """Method called in init process, overload it! + + this method will be called before reading config from + file/args/kwargs + """ + # for name in ['rotation']: + # setattr(self.__class__, name, _configPropertyRad(name+'d')) + # self._configlist['Experiment'].extend(['rotation']) + return + + ########################################################################### + + def _findConfigFile(self, filename=None, args=None, **kwargs): + """Find config file, if any config is specified in + filename/args/kwargs then return the filename of config. + + :param filename: str, file name of config file + :param filename: list of str, args passed from cmd + :param kwargs: optional kwargs + :return: name of config file if found, otherwise None + """ + rv = None + if filename is not None: + rv = filename + if args is not None: + if ("--configfile" in args) or ("-c" in args): + obj = self.args.parse_args(args) + rv = obj.configfile + if "configfile" in kwargs: + rv = kwargs["configfile"] + return rv + + def _findDefaultConfigFile(self, filename=None, args=None, **kwargs): + """Find default config file, if any config is specified in + filename/args/kwargs or in self._defaultdata['configfile'], then + return the filename of config. + + kwargs > args > filename > default + + param filename: str, file name of config file + param filename: list of str, args passed from cmd + param kwargs: optional kwargs + + return: name of config file if found, otherwise None + """ + rv = self._findConfigFile(filename, args, **kwargs) + if rv is None: + for dconf in self._defaultdata["configfile"]: + if (os.path.exists(dconf)) and (rv is None): + rv = dconf + return rv + + ########################################################################### + + def _updateSelf(self, optnames=None, **kwargs): + """Update the options value, then copy the values in the + self.'options' to self.config. + + 1. call self._preUpdateSelf + 2. apply options' value from *self.option* to self.config + 3. call self._postUpdateSelf + + :param optnames: str or list of str, name of options whose value has + been changed, if None, update all options + """ + # so some check right here + self._preUpdateSelf(**kwargs) + # copy value to self.config + self._copySelftoConfig(optnames) + # so some check right here + self._postUpdateSelf(**kwargs) + return + + # example, overload it + def _preUpdateSelf(self, **kwargs): + """Additional process called in self._updateSelf, this method is + called before self._copySelftoConfig(), i.e. before copy options + value to self.config (config file)""" + return + + def _postUpdateSelf(self, **kwargs): + """Additional process called in self._updateSelf, this method is + called after self._copySelftoConfig(), i.e. before copy options + value to self.config (config file)""" + return + + ########################################################################### + + def _getTypeStr(self, optname): + """Return the type of option. + + :param optname: str, name of option + :return: string, type of the option + """ + opttype = self._getTypeStrC(optname) + return opttype + + @classmethod + def _getTypeStrC(cls, optname): + """Class method, return the type of option first try to get type + information from metadata, if failed, try to get type from + default value. + + :param optname: str, name of option + :return: string, type of the option + """ + optdata = cls._optdata[optname] + if "t" in optdata: + opttype = optdata["t"] + else: + value = optdata["d"] + if isinstance(value, str): + opttype = "str" + elif isinstance(value, bool): + opttype = "bool" + elif isinstance(value, float): + opttype = "float" + elif isinstance(value, int): + opttype = "int" + elif isinstance(value, list): + if len(value) == 0: + opttype = "strlist" + elif isinstance(value[0], str): + opttype = "strlist" + elif isinstance(value[0], bool): + opttype = "boollist" + elif isinstance(value[0], float): + opttype = "floatlist" + elif isinstance(value[0], int): + opttype = "intlist" + + return opttype + + ########################################################################### + + def _detectAddSections(self): + """Detect sections present in self._optdata and add them to + self.config also add it to self._configlist.""" + self._detectAddSectionsC(self) + return + + @classmethod + def _detectAddSectionsC(cls): + """Class method, detect sections present in self._optdata and + add them to self.config also add it to self._configlist.""" + # seclist = [self._optdata[key]['sec'] for key in self._optdata.keys()] + seclist = [cls._optdata[opt[0]]["sec"] for opt in cls._optdatalist] + secdict = OrderedDict.fromkeys(seclist) + # for sec in set(seclist): + for sec in secdict.keys(): + cls.config.add_section(sec) + cls._configlist[sec] = [] + return + + def _addOpt(self, optname): + """Add options to self.config and self.args and self.*option*, + this will read metadata from self._optdatalist. + + :param optname: string, name of option + """ + self._addOptC(self, optname) + return + + @classmethod + def _addOptC(cls, optname): + """Class method, add options to self.config and self.args and + self.*option*, this will read metadata in self._optdatalist. + + :param optname: string, name of option + """ + optdata = cls._optdata[optname] + opttype = cls._getTypeStrC(optname) + + # replace currentdir in default to os.getcwd() + if optdata["d"] == "currentdir": + optdata["d"] = os.getcwd() + + # add to cls.'optname' + cls._addOptSelfC(optname, optdata) + + # add to cls.config + secname = optdata["sec"] if "sec" in optdata else "Others" + cls._configlist[secname].append(optname) + if optdata.get("config", "a") != "n": + strvalue = ( + ", ".join(map(str, optdata["d"])) + if isinstance(optdata["d"], list) + else str(optdata["d"]) + ) + cls.config.set(secname, optname, strvalue) + # add to cls.args + if optdata.get("args", "a") != "n": + # transform optdata to a dict that can pass to add_argument method + pargs = dict() + for key in optdata.keys(): + if key in cls._optdatanamedict: + pargs[cls._optdatanamedict[key]] = optdata[key] + pargs["default"] = argparse.SUPPRESS + pargs["type"] = StrConv(opttype) + # add args + if "f" in optdata: + cls.args.add_argument(optname, **pargs) + elif "s" in optdata: + cls.args.add_argument( + "--" + optname, "-" + optdata["s"], **pargs + ) + else: + cls.args.add_argument("--" + optname, **pargs) + return + + @classmethod + def _addOptSelfC(cls, optname, optdata): + """Class method, assign options value to *self.option*, using + metadata. + + :param optname: string, name of the option + :param optdata: dict, metadata of the options, get it from + self._optdatalist + """ + setattr(cls, optname, optdata["d"]) + return + + def _copyConfigtoSelf(self, optnames=None): + """Copy the options' value from self.config to self.*option* + + :param optnames: str or list of str, names of options whose + value copied from self.config to self.*option*'. Set None to + update all + """ + if optnames is not None: + optnames = optnames if isinstance(optnames, list) else [optnames] + else: + optnames = [] + for secname in self.config.sections(): + optnames += self.config.options(secname) + + for optname in optnames: + if optname in self._optdata: + secname = self._optdata[optname]["sec"] + opttype = self._getTypeStr(optname) + optvalue = self.config.get(secname, optname) + setattr(self, optname, str2Opt(opttype, optvalue)) + return + + def _copySelftoConfig(self, optnames=None): + """Copy the value from self.*option* to self.config. + + :param optname: str or list of str, names of options whose value + copied from self.*option* to self.config. Set None to update + all + """ + if optnames is not None: + optnames = optnames if isinstance(optnames, list) else [optnames] + else: + optnames = [] + for secname in self.config.sections(): + optnames += self.config.options(secname) + + for optname in optnames: + if optname in self._optdata: + secname = self._optdata[optname]["sec"] + opttype = self._getTypeStr(optname) + optvalue = getattr(self, optname) + self.config.set(secname, optname, opt2Str(opttype, optvalue)) + return + + ########################################################################### + + def parseArgs(self, pargs): + """Parse args and update the value in self.*option*, this will + call the self.args() to parse args, + + :param pargs: list of string, arguments to parse, usually coming + from sys.argv + """ + obj = self.args.parse_args(pargs) + changedargs = obj.__dict__.keys() + for optname in changedargs: + if optname in self._optdata: + setattr(self, optname, getattr(obj, optname)) + # update self + if len(changedargs) > 0: + self._updateSelf(changedargs) + return obj + + def parseKwargs(self, **kwargs): + """Update self.*option* values according to the kwargs. + + :param kwargs: dict, keywords=value + """ + if kwargs != {}: + changedargs = [] + for optname, optvalue in kwargs.items(): + if optname in self._optdata: + setattr(self, optname, optvalue) + changedargs.append(optname) + # update self + self._updateSelf(changedargs) + return + + def parseConfigFile(self, filename): + """Read a config file and update the self.*option* + + :param filename: str, file name of config file (include path) + """ + if filename is not None: + filename = os.path.abspath(filename) + if os.path.exists(filename): + self.configfile = filename + self._copySelftoConfig() + fileobj = FakeConfigFile(filename) + # self.config.read(filename) + self.config.readfp(fileobj) + self._copyConfigtoSelf() + self._updateSelf() + return + + def updateConfig(self, filename=None, args=None, **kwargs): + """Update config according to config file, args(from sys.argv) + or **kwargs. + + 1. call self._preUpdateConfig() + 2. process file/args/kwargs passed to this method, + 3. read a configfile if specified in args or kwargs + 4. call self._postUpdateConfig() + 5. write config file if specified in args/kwargs + + :param filename: str, file name of the config file + :param args: list of str, args passed from cmd, + :param kwargs: dict, optional kwargs + + :return: True if anything updated, False if nothing updated + """ + # call self._preUpdateConfig + self._preUpdateConfig(**kwargs) + + filename = self._findConfigFile(filename, args, **kwargs) + if filename is not None: + rv = self.parseConfigFile(filename) + if args is not None: + rv = self.parseArgs(args) + if kwargs != {}: + rv = self.parseKwargs(**kwargs) + + if ( + (filename is None) + and ((args is None) or (args == [])) + and (kwargs == {}) + ): + rv = self._updateSelf() + + # call self._callbackUpdateConfig + self._postUpdateConfig(**kwargs) + + # write config file + self._createConfigFile() + return rv + + def _preUpdateConfig(self, **kwargs): + """Method called before parsing args or kwargs or config file, + in self.updateConfig.""" + return + + def _postUpdateConfig(self, **kwargs): + """Method called after parsing args or kwargs or config file, in + self.updateConfig.""" + return + + ########################################################################### + def _createConfigFile(self): + """Write output config file if specified in configuration the + filename is specified by self.createconfig.""" + if (self.createconfig != "") and (self.createconfig is not None): + self.writeConfig(self.createconfig, "short") + self.createconfig = "" + if (self.createconfigfull != "") and ( + self.createconfigfull is not None + ): + self.writeConfig(self.createconfigfull, "full") + self.createconfigfull = "" + return + + def writeConfig(self, filename, mode="short", changeconfigfile=True): + """Write config to file. the file is compatible with python + package ConfigParser. + + :param filename: string, name of file + :param mode: string, 'short' or 'full' ('s' or 'f'). in short + mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + """ + if changeconfigfile: + self.configfile = os.path.abspath(filename) + self._updateSelf() + # func decide if write the option to config according to mode + # options not present in self._optdata will not be written to config + if mode.startswith("s"): + mcond = ( + lambda optname: self._optdata.get( + optname, {"config": "n"} + ).get("config", "a") + == "a" + ) + else: + mcond = ( + lambda optname: self._optdata.get( + optname, {"config": "n"} + ).get("config", "a") + != "n" + ) + + lines = [] + for section in self.config._sections: + tlines = [] + for key, value in self.config._sections[section].items(): + if (key != "__name__") and mcond(key): + tlines.append( + "%s = %s" % (key, str(value).replace("\n", "\n\t")) + ) + if len(tlines) > 0: + lines.append("[%s]" % section) + lines.extend(tlines) + lines.append("") + rv = "\n".join(lines) + "\n" + fp = open(filename, "w") + fp.write(rv) + fp.close() + return + + def getHeader(self, title=None, mode="full"): + """Get a header of configurations values, + + :param title: str, title of header, if None, try to get it from + self.defaultvalue + :param mode: string, 'short' or 'full' ('s' or 'f'). in short + mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + :return: string, lines with line break that can be directly + written to a text file + """ + + lines = [] + title = "# %s #" % ( + self._defaultdata["headertitle"] if title is None else title + ) + lines.append(title) + # func decide if write the option to header according to mode + # options not present in self._optdata will not be written to header + if mode.startswith("s"): + mcond = ( + lambda optname: self._optdata.get( + optname, {"header": "n"} + ).get("header", "a") + == "a" + ) + else: + mcond = ( + lambda optname: self._optdata.get( + optname, {"header": "n"} + ).get("header", "a") + != "n" + ) + + for secname in self._configlist.keys(): + tlines = [] + for optname in self._configlist[secname]: + if mcond(optname): + value = getattr(self, optname) + ttype = self._getTypeStr(optname) + strvalue = ( + ", ".join(map(str, value)) + if ttype.endswith("list") + else str(value) + ) + tlines.append("%s = %s" % (optname, strvalue)) + if len(tlines) > 0: + lines.append("[%s]" % secname) + lines.extend(tlines) + lines.append("") + rv = "\n".join(lines) + "\n" + return rv + + def resetDefault(self, optnames=None): + """Reset all values to their default value. + + :param optnames: list of str, name of options to reset, None for + all options + """ + if optnames is None: + optnames = self._optdata.keys() + for optname in optnames: + if optname in self._optdata: + setattr(self, optname, self._optdata[optname]["d"]) + self._updateSelf() + return + + ########################################################################### + # IMPORTANT call this method if you want to add + # options as class attributes!!! + + @classmethod + def initConfigClass(cls): + """Init config class and add options to class. + + IMPORTANT call this method after you define the metadata of your + config class to add options as class attributes!!! + """ + cls._preInitConfigClass() + + cls.config = ConfigParser(dict_type=OrderedDict) + cls.args = argparse.ArgumentParser( + description=cls._description, + epilog=cls._epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + cls._configlist = OrderedDict({}) + + cls._optdatalist = cls._optdatalist_default + cls._optdatalist + cls._optdata = dict(cls._optdatalist) + cls._detectAddSectionsC() + for opt in cls._optdatalist: + key = opt[0] + cls._addOptC(key) + + cls._postInitConfigClass() + return + + @classmethod + def _postInitConfigClass(cls): + """Additional processes called after initConfigClass. + + overload it + """ + pass + + @classmethod + def _preInitConfigClass(cls): + """Additional processes called before initConfigClass. + + overload it + """ + pass + + +# VERY IMPORTANT!!! +# add options to class +# initConfigClass(ConfigBase) +# ConfigBase.initConfigClass() + +if __name__ == "__main__": + + test = ConfigBase() + test.updateConfig() diff --git a/src/diffpy/srxconfutils/configtraits.py b/src/diffpy/srxconfutils/configtraits.py new file mode 100644 index 0000000..9e90e29 --- /dev/null +++ b/src/diffpy/srxconfutils/configtraits.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.srxconfutils by Simon J. L. Billinge group +# (c) 2013-2025 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.rst for a list of people who contributed. +# See LICENSENOTICE.rst for license information. +# +############################################################################## +"""Package for organizing program configurations. It can read/write +configurations file, parse arguments from command lines, and also parse +arguments passed from method/function calling inside python. + +This one is similar to ConfigBase but use Traits, so every option +(self.*option* is a trait) can be observed and have a GUI interface. +""" + +from traits.api import ( + Array, + Bool, + CFloat, + CInt, + Directory, + Enum, + File, + HasTraits, + List, + String, +) + +from diffpy.srxconfutils.config import ConfigBase + + +class ConfigBaseTraits(HasTraits, ConfigBase): + """_optdatalist_default, _optdatalist are metadata used to + initialize the options, see below for examples. + + options presents in --help (in cmd), config file, headers have + same order as in these list, so arrange them in right order here. + + optional args to control if the options presents in args, config file or + file header + + 'args' - default is 'a' + if 'a', this option will be available in self.args + if 'n', this option will not be available in self.args + 'config' - default is 'a' + if 'f', this option will present in self.config and be written to + config file only in full mode + if 'a', this option will present in self.config and be written to + config file both in full and short mode + if 'n', this option will not present in self.config + 'header' - default is 'a' + if 'f', this option will be written to header only in full mode + if 'a', this option will be written to header both in full and short + mode + if 'n', this option will not be written to header + + so in short mode, all options with 'a' will be written, in full mode, + all options with 'a' or 'f' will be written + """ + + # Text to display before the argument help + _description = """Description of configurations + """ + # Text to display after the argument help + _epilog = """ + """ + + """ + optdata contains these keys: + these args will be passed to argparse, see the documents of argparse for + detail information + + 'f': full, (positional) + 's': short + 'h': help + 't': type + 'a': action + 'n': nargs + 'd': default + 'c': choices + 'r': required + 'de': dest + 'co': const + + additional options for traits: + 'tt': traits type + 'l': traits label + """ + _optdatanamedict = { + "h": "help", + "t": "type", + "a": "action", + "n": "nargs", + "d": "default", + "c": "choices", + "r": "required", + "de": "dest", + "co": "const", + } + _traitstypedict = { + "str": String, + "int": CInt, + "float": CFloat, + "bool": Bool, + "file": File, + "directory": Directory, + "strlist": List, + "intlist": List, + "floatlist": List, + "boollist": List, + "array": Array, + } + + # examples, overload it + _optdatalist_default = [ + [ + "configfile", + { + "sec": "Control", + "config": "f", + "header": "n", + "l": "Config File", + "tt": "file", + "s": "c", + "h": "name of input config file", + "d": "", + }, + ], + [ + "createconfig", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": ( + "create a config file according to " + "default or current values" + ), + "d": "", + }, + ], + [ + "createconfigfull", + { + "sec": "Control", + "config": "n", + "header": "n", + "h": "create a full configurable config file", + "d": "", + }, + ], + ] + # examples, overload it + _optdatalist = [ + [ + "tifdirectory", + { + "sec": "Experiment", + "header": "n", + "tt": "directory", + "l": "Tif directory", + "s": "tifdir", + "h": "directory of raw tif files", + "d": "currentdir", + }, + ], + [ + "integrationspace", + { + "sec": "Experiment", + "l": "Integration space", + "h": "integration space, could be twotheta or qspace", + "d": "twotheta", + "c": ["twotheta", "qspace"], + }, + ], + [ + "wavelength", + { + "sec": "Experiment", + "l": "Wavelength", + "h": "wavelength of x-ray, in A", + "d": 0.1000, + }, + ], + [ + "rotationd", + { + "sec": "Experiment", + "l": "Tilt Rotation", + "s": "rot", + "h": "rotation angle of tilt plane, in degree", + "d": 0.0, + }, + ], + [ + "includepattern", + { + "sec": "Beamline", + "header": "n", + "config": "f", + "l": "Include", + "s": "ipattern", + "h": "file name pattern for included files", + "n": "*", + "d": ["*.tif"], + }, + ], + [ + "excludepattern", + { + "sec": "Beamline", + "header": "n", + "config": "f", + "l": "Exclude", + "s": "epattern", + "h": "file name pattern for excluded files", + "n": "*", + "d": ["*.dark.tif", "*.raw.tif"], + }, + ], + [ + "fliphorizontal", + { + "sec": "Beamline", + "header": "n", + "config": "f", + "l": "Flip horizontally", + "h": "flip the image horizontally", + "n": "?", + "co": True, + "d": False, + }, + ], + [ + "maskedges", + { + "sec": "Others", + "config": "f", + "tt": "array", + "l": "Mask edges", + "h": ( + "mask the edge pixels, first four means " + "the number of pixels masked in each edge " + "(left, right, top, bottom), the last one is the " + "radius of a region masked around the corner" + ), + "n": 5, + "d": [10, 10, 10, 10, 100], + }, + ], + ] + + # default config file path and name + _defaultdata = { + "configfile": ["config.cfg"], + "headertitle": "Configuration information", + } + + def __init__(self, filename=None, args=None, **kwargs): + """Init the class and update the values of options if specified + in filename/args/kwargs. + + it will: + 1. init class using HasTraits + 2. call self._preInit method + 3. find the config file if specified in filename/args/kwargs + if failed, try to find default config file + 4. update the options value using filename/args/kwargs + file > args > kwargs + 5. call self._postInitTraits() + + :param filename: str, file name of the config file + :param args: list of str, args passed from cmd + :param kwargs: dict, optional kwargs + + :return: None + """ + HasTraits.__init__(self) + ConfigBase.__init__(self, filename, args, **kwargs) + + self._postInitTraits() + return + + def _postInitTraits(self): + """Additional init process called after traits init.""" + return + + @classmethod + def _addOptSelfC(cls, optname, optdata): + """Class method, assign options value to *self.option*, using + metadata, this one will create traits objects for each option. + + :param optname: string, name of the option + :param optdata: dict, metadata of the options, get it from + self._optdatalist + """ + # value type + vtype = cls._getTypeStrC(optname) + ttype = optdata.get("tt", vtype) + ttype = cls._traitstypedict[ttype] + kwargs = { + "label": optdata["l"] if "l" in optdata else optname, + "desc": optdata["h"], + } + args = [optdata["d"]] + if "c" in optdata: + ttype = Enum + args = [optdata["c"]] + kwargs["value"] = optdata["d"] + if ttype == Array: + args = [] + kwargs["value"] = optdata["d"] + obj = ttype(*args, **kwargs) + cls.add_class_trait(optname, obj) + return + + +# ConfigBaseTraits.initConfigClass() + +if __name__ == "__main__": + test = ConfigBaseTraits(filename="temp.cfg") + test.updateConfig() + test.configure_traits() diff --git a/dpx/confutils/gitarchive.cfg b/src/diffpy/srxconfutils/gitarchive.cfg similarity index 76% rename from dpx/confutils/gitarchive.cfg rename to src/diffpy/srxconfutils/gitarchive.cfg index 2f8fb70..43b349f 100644 --- a/dpx/confutils/gitarchive.cfg +++ b/src/diffpy/srxconfutils/gitarchive.cfg @@ -2,4 +2,4 @@ commit = $Format:%H$ date = $Format:%ai$ timestamp = $Format:%at$ -refnames = $Format:%D$ \ No newline at end of file +refnames = $Format:%D$ diff --git a/src/diffpy/srxconfutils/srxconfutils_app.py b/src/diffpy/srxconfutils/srxconfutils_app.py new file mode 100644 index 0000000..f07f27b --- /dev/null +++ b/src/diffpy/srxconfutils/srxconfutils_app.py @@ -0,0 +1,33 @@ +import argparse + +from diffpy.srxconfutils.version import __version__ # noqa + + +def main(): + parser = argparse.ArgumentParser( + prog="diffpy.srxconfutils", + description=( + "Configuration utilities for dpx project. Part of xPDFsuite\n\n" + "For more information, visit: " + "https://github.com/diffpy/diffpy.srxconfutils/" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--version", + action="store_true", + help="Show the program's version number and exit", + ) + + args = parser.parse_args() + + if args.version: + print(f"diffpy.srxconfutils {__version__}") + else: + # Default behavior when no arguments are given + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/src/diffpy/srxconfutils/tools.py b/src/diffpy/srxconfutils/tools.py new file mode 100644 index 0000000..9b8a503 --- /dev/null +++ b/src/diffpy/srxconfutils/tools.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python +############################################################################## +# +# diffpy.srxconfutils by Simon J. L. Billinge group +# (c) 2013-2025 Trustees of the Columbia University +# in the City of New York. All rights reserved. +# +# File coded by: Xiaohao Yang +# +# See AUTHORS.rst for a list of people who contributed. +# See LICENSENOTICE.rst for license information. +# +############################################################################## + +import hashlib +import re +import time +import zlib +from pkgutil import iter_modules + +import numpy as np + + +def module_exists(module_name): + return module_name in [tuple_[1] for tuple_ in iter_modules()] + + +def module_exists_lower(module_name): + return module_name.lower() in [ + tuple_[1].lower() for tuple_ in iter_modules() + ] + + +def _configPropertyRad(nm): + """Helper function of options delegation, rad to degree.""" + rv = property( + fget=lambda self: np.radians(getattr(self, nm)), + fset=lambda self, val: setattr(self, nm, np.degrees(val)), + fdel=lambda self: delattr(self, nm), + ) + return rv + + +def _configPropertyR(name): + """Create a property that forwards self.name to self.config.name. + + read only + """ + rv = property( + fget=lambda self: getattr(self.config, name), + doc="attribute forwarded to self.config, read-only", + ) + return rv + + +def _configPropertyRW(name): + """Create a property that forwards self.name to self.config.name. + + read and write + """ + rv = property( + fget=lambda self: getattr(self.config, name), + fset=lambda self, value: setattr(self.config, name, value), + fdel=lambda self: delattr(self, name), + doc="attribute forwarded to self.config, read/write", + ) + return rv + + +def str2bool(v): + """Turn string to bool.""" + return v.lower() in ("yes", "true", "t", "1") + + +def opt2Str(opttype, optvalue): + """Turn the value of one option to string, according to the option + type list of values are turned into "value1, value2, value3...". + + :param opttype: string, type of options, for example 'str' or + 'intlist' + :param optvalue: value of the option + :return: string, usually stored in ConfigBase.config + """ + + if opttype.endswith("list"): + rv = ", ".join(map(str, optvalue)) + else: + rv = str(optvalue) + return rv + + +def StrConv(opttype): + """Get the type (or converter function) according to the opttype. + + the function doesn't take list + """ + if opttype.startswith("str"): + conv = str + elif opttype.startswith("int"): + conv = int + elif opttype.startswith("float"): + conv = float + elif opttype.startswith("bool"): + conv = str2bool + else: + conv = None + return conv + + +def str2Opt(opttype, optvalue): + """Convert the string to value of one option, according to the + option type. + + :param opttype: string, type of options, for example 'str' or + 'intlist' + :param optvalue: string, value of the option + :return: value of the option, usually stored in ConfigBase.config + """ + # base converter + conv = StrConv(opttype) + if opttype.endswith("list"): + temp = re.split(r"\s*,\s*", optvalue) + rv = list(map(conv, temp)) if len(temp) > 0 else [] + else: + rv = conv(optvalue) + return rv + + +class FakeConfigFile(object): + """A fake configfile object used in reading config from header of + data or a real config file.""" + + def __init__(self, configfile, endline="###"): + self.configfile = configfile + self.fp = open(configfile) + self.endline = endline + self.ended = False + self.name = configfile + return + + def readline(self): + """Readline function.""" + line = self.fp.readline() + if line.startswith(self.endline) or self.ended: + rv = "" + self.ended = True + else: + rv = line + return rv + + def close(self): + """Close the file.""" + self.fp.close() + return + + def __iter__(self): + return self + + def __next__(self): + line = self.readline() + if line == "": + raise StopIteration + return line + + +def get_crc32(filename): + """Calculate the crc32 value of file. + + :param filename: path to the file + :return: crc32 value of file + """ + try: + with open(filename, "rb") as fd: + eachLine = fd.readline() + prev = 0 + while eachLine: + prev = zlib.crc32(eachLine, prev) + eachLine = fd.readline() + except OSError as e: + raise RuntimeError(f"Failed to read file {filename}") from e + return prev + + +def get_md5(filename, blocksize=65536): + """Calculate the MD5 value of file. + + :param filename: path to the file + :return: md5 value of file + """ + try: + with open(filename, "rb") as fd: + buf = fd.read(blocksize) + md5 = hashlib.md5() + while len(buf) > 0: + md5.update(buf) + buf = fd.read(blocksize) + except OSError as e: + raise RuntimeError(f"Failed to read file {filename}") from e + return md5.hexdigest() + + +def checkFileVal(filename): + """Check file integrity using crc32 and md5. It will read file twice + then compare the crc32 and md5. If two results doesn't match, it + will wait until the file is completed written to disk. + + :param filename: path to the file + """ + valflag = False + lastcrc = get_crc32(filename) + while not valflag: + currcrc = get_crc32(filename) + if currcrc == lastcrc: + lastmd5 = get_md5(filename) + time.sleep(0.01) + currmd5 = get_md5(filename) + if lastmd5 == currmd5: + valflag = True + else: + time.sleep(0.5) + lastcrc = get_crc32(filename) + return diff --git a/src/diffpy/srxconfutils/version.py b/src/diffpy/srxconfutils/version.py new file mode 100644 index 0000000..dbd80dd --- /dev/null +++ b/src/diffpy/srxconfutils/version.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +############################################################################## +# +# (c) 2013-2025 The Trustees of Columbia University in the City of New York. +# All rights reserved. +# +# File coded by: Xiaohao Yang and Billinge Group members. +# +# See GitHub contributions for a more detailed list of contributors. +# https://github.com/diffpy/diffpy.srxconfutils/graphs/contributors # noqa: E501 +# +# See LICENSE.rst for license information. +# +############################################################################## +"""Definition of __version__.""" + +# We do not use the other three variables, but can be added back if needed. +# __all__ = ["__date__", "__git_commit__", "__timestamp__", "__version__"] + +# obtain version information +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("diffpy.srxconfutils") +except PackageNotFoundError: + __version__ = "unknown" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a3114ea --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +import json +from pathlib import Path + +import pytest + + +@pytest.fixture +def user_filesystem(tmp_path): + base_dir = Path(tmp_path) + home_dir = base_dir / "home_dir" + home_dir.mkdir(parents=True, exist_ok=True) + cwd_dir = base_dir / "cwd_dir" + cwd_dir.mkdir(parents=True, exist_ok=True) + + home_config_data = {"username": "home_username", "email": "home@email.com"} + with open(home_dir / "diffpyconfig.json", "w") as f: + json.dump(home_config_data, f) + + yield tmp_path + + +@pytest.fixture +def temp_file(tmp_path): + """Create a temporary file with known content.""" + file_path = tmp_path / "testfile.txt" + content = b"Hello world!\nThis is a test.\n" + file_path.write_bytes(content) + return file_path, content diff --git a/tests/test_tools.py b/tests/test_tools.py new file mode 100644 index 0000000..864fdd4 --- /dev/null +++ b/tests/test_tools.py @@ -0,0 +1,19 @@ +import hashlib +import zlib + +from diffpy.srxconfutils import tools + + +def test_get_md5(temp_file): + file_path, content = temp_file + expected_md5 = hashlib.md5(content).hexdigest() + result = tools.get_md5(file_path) + assert result == expected_md5 + + +def test_get_crc32(temp_file): + """Test the get_crc32 function.""" + file_path, content = temp_file + val = tools.get_crc32(file_path) + expected = zlib.crc32(content) + assert val == expected diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..e2d2c24 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,10 @@ +"""Unit tests for __version__.py.""" + +import diffpy.srxconfutils # noqa + + +def test_package_version(): + """Ensure the package version is defined and not set to the initial + placeholder.""" + assert hasattr(diffpy.srxconfutils, "__version__") + assert diffpy.srxconfutils.__version__ != "0.0.0"