diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 4ad0ecf..0000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -tests/measurment-13-node-120s.json filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index f6aa9e5..004972a 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -1,216 +1,185 @@ -name: Deploy Dev Release Artifacts - -on: - push: - branches: - - develop - workflow_dispatch: - inputs: - release-version: - description: "Version number to use. If provided bump-rule will be ignored" - required: false - default: "" - type: string - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYTHON_VERSION: "3.10" - -jobs: - deploy-dev-release: - runs-on: ubuntu-22.04 - permissions: - contents: write # To push a branch - pull-requests: write # To create a PR from that branch - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - # ref: develop - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1.3.3 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - # #---------------------------------------------- - # # load cached venv if cache exists - # #---------------------------------------------- - # - name: Load cached venv - # id: cached-poetry-dependencies - # uses: actions/cache@v3 - # with: - # path: .venv - # key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - # #---------------------------------------------- - # # install dependencies if cache does not exist - # #---------------------------------------------- - # - name: Install dependencies - # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - # run: poetry install --no-interaction --no-root - - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: | - ./scripts/poetry_install.sh - - # git checkout develop - # poetry lock --no-update - # poetry install --no-interaction - - # - name: Use given release-version number - # if: inputs.release-version != '' - # run: | - # echo "Using given release version is ${{ inputs.release-version }}" - # poetry version ${{ inputs.release-version }} - - # NEW_TAG=v$(poetry version --short) - - # # we want to be able to use the variable in later - # # steps we set a NEW_TAG environmental variable - # echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - # # we don't want to update pyproject.toml yet. don't want this change to create merge conflict. - # # we don't really persist right version in pyproject.toml to figure out the next version. we use git tags. - # git restore pyproject.toml - - #---------------------------------------------- - # bump version number for patch - #---------------------------------------------- - - name: Bump Version - run: | - # current_tag is the last tagged release in the repository. From there - # we need to remove the v from the beginning of the tag. - # echo "Bump rule is ${{ inputs.bump-rule }}" - # echo "Given release version is ${{ inputs.release-version }}" - dt=$(date +%Y.%-m.0) - if ! $(git tag -l "v*" = ''); then - # uses -V which is version sort to keep it monotonically increasing. - current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) - echo "current git tag is ${current_tag}" - current_tag=${current_tag#?} - if [[ "$current_tag" < "$dt" ]]; then - current_tag=$dt - fi - # current_tag is now the version we want to set our poetry version so - # that we can bump the version - ./scripts/run_on_each.sh poetry version ${current_tag} - ./scripts/run_on_each.sh poetry version prerelease - - # poetry version ${current_tag} - # poetry version prerelease --no-interaction - - else - # very first release. start with inputs.release-version - - echo "First release. Setting tag as 0.1.0rc0" - current_tag=$(date +%Y.%-m.1) - ./scripts/run_on_each.sh poetry version ${current_tag} - - # poetry version ${current_tag} - fi - - NEW_TAG=v$(poetry version --short) - - # Finally because we want to be able to use the variable in later - # steps we set a NEW_TAG environmental variable - echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - - - name: Create build artifacts - run: | - set -x - set -u - set -e - - # set the right version in pyproject.toml before build and publish - ./scripts/poetry_build.sh - - - name: Push artifacts to github - uses: ncipollo/release-action@v1 - with: - artifacts: "dist/*.gz,dist/*.whl" - artifactErrorsFailBuild: true - generateReleaseNotes: true - commit: ${{ github.ref }} - # check bump-rule and set accordingly - prerelease: true - tag: ${{ env.NEW_TAG }} - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish to pypi - id: publish-to-pypi - if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' - run: | - set -x - set -u - set -e - - # This is needed, because the poetry publish will fail at the top level of the project - # so ./scripts/run_on_each.sh fails for that. - echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - cd gridappsd-python-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish - - cd ../gridappsd-field-bus-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish - - # - name: Publish to test-pypi - # id: publish-to-test-pypi - # if: ${{ inputs.publish-to-test-pypi }} - # run: | - # poetry config repositories.test-pypi https://test.pypi.org/legacy/ - # poetry config pypi-token.test-pypi ${{ secrets.pypi-token }} - # poetry publish -r test-pypi - # continue-on-error: true - - # - name: if publish to pypi/test-pypi failed revert main and delete release branch - # if: ${{ steps.publish-to-pypi.outcome != 'success' && steps.publish-to-test-pypi.outcome != 'success' }} - # run: | - # echo "publish to pypi/test-pypi did not succeed. Outcome for pypi = ${{ steps.publish-to-pypi.outcome }} outcome for test-pypi= ${{ steps.publish-to-test-pypi.outcome }}" - # git reset --hard HEAD~1 - # git push origin HEAD --force - # git branch -d releases/${NEW_TAG} - # git push origin --delete releases/${NEW_TAG} - # echo "reverted changes to main and removed release branch" - - # - name: if publish to pypi/test-pypi failed delete release and tag on github - # if: ${{ ! (steps.publish-to-pypi.outcome == 'success' || steps.publish-to-test-pypi.outcome == 'success') }} - # uses: dev-drprasad/delete-tag-and-release@v0.2.1 - # env: - # GITHUB_TOKEN: ${{ secrets.git-token }} - # with: - # tag_name: ${{ env.NEW_TAG }} - - # - name: if publish to pypi/test-pypi failed exit with exit code 1 - # if: ${{ steps.publish-to-pypi.outcome != 'success' && steps.publish-to-test-pypi.outcome != 'success' }} - # run: | - # exit 1 \ No newline at end of file +name: Deploy Dev Release Artifacts + +on: + push: + branches: + - develop + workflow_dispatch: + inputs: + release-version: + description: "Version number to use. If provided bump-rule will be ignored" + required: false + default: "" + type: string + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: "3.10" + +jobs: + deploy-dev-release: + runs-on: ubuntu-22.04 + permissions: + contents: write # To push a branch + pull-requests: write # To create a PR from that branch + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + # ref: develop + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1.3.3 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + # #---------------------------------------------- + # # load cached venv if cache exists + # #---------------------------------------------- + # - name: Load cached venv + # id: cached-poetry-dependencies + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + # #---------------------------------------------- + # # install dependencies if cache does not exist + # #---------------------------------------------- + # - name: Install dependencies + # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # run: poetry install --no-interaction --no-root + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: | + ./scripts/poetry_install.sh + + # git checkout develop + # poetry lock --no-update + # poetry install --no-interaction + + # - name: Use given release-version number + # if: inputs.release-version != '' + # run: | + # echo "Using given release version is ${{ inputs.release-version }}" + # poetry version ${{ inputs.release-version }} + + # NEW_TAG=v$(poetry version --short) + + # # we want to be able to use the variable in later + # # steps we set a NEW_TAG environmental variable + # echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + # # we don't want to update pyproject.toml yet. don't want this change to create merge conflict. + # # we don't really persist right version in pyproject.toml to figure out the next version. we use git tags. + # git restore pyproject.toml + + #---------------------------------------------- + # bump version number for patch + #---------------------------------------------- + - name: Bump Version + run: | + # current_tag is the last tagged release in the repository. From there + # we need to remove the v from the beginning of the tag. + # echo "Bump rule is ${{ inputs.bump-rule }}" + # echo "Given release version is ${{ inputs.release-version }}" + dt=$(date +%Y.%-m.0) + if ! $(git tag -l "v*" = ''); then + # uses -V which is version sort to keep it monotonically increasing. + current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) + echo "current git tag is ${current_tag}" + current_tag=${current_tag#?} + if [[ "$current_tag" < "$dt" ]]; then + current_tag=$dt + fi + # current_tag is now the version we want to set our poetry version so + # that we can bump the version + ./scripts/run_on_each.sh poetry version ${current_tag} + ./scripts/run_on_each.sh poetry version prerelease + + # poetry version ${current_tag} + # poetry version prerelease --no-interaction + + else + # very first release. start with inputs.release-version + + echo "First release. Setting tag as 0.1.0rc0" + current_tag=$(date +%Y.%-m.1) + ./scripts/run_on_each.sh poetry version ${current_tag} + + # poetry version ${current_tag} + fi + + NEW_TAG=v$(poetry version --short) + + # Finally because we want to be able to use the variable in later + # steps we set a NEW_TAG environmental variable + echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + + - name: Create build artifacts + run: | + set -x + set -u + set -e + + # set the right version in pyproject.toml before build and publish + ./scripts/poetry_build.sh + + - name: Push artifacts to github + uses: ncipollo/release-action@v1 + with: + artifacts: "dist/*.gz,dist/*.whl" + artifactErrorsFailBuild: true + generateReleaseNotes: true + commit: ${{ github.ref }} + # check bump-rule and set accordingly + prerelease: true + tag: ${{ env.NEW_TAG }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to pypi + id: publish-to-pypi + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' + run: | + set -x + set -u + set -e + + # This is needed, because the poetry publish will fail at the top level of the project + # so ./scripts/run_on_each.sh fails for that. + echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish + + cd ../gridappsd-field-bus-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish + diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml deleted file mode 100644 index b46e965..0000000 --- a/.github/workflows/dev-release.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -name: Deploy Pre-Release Artifacts - -on: - push: - branches: - - develop - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.8' - RUNS_ON: ubuntu-latest - -jobs: - - bump_version: - runs-on: ubuntu-latest - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2.1.7 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: | - poetry install --no-interaction - - #---------------------------------------------- - # bump version number for patch - #---------------------------------------------- - - name: Bump Version - run: | - - # $versionarr now holds the current version of the tag. - IFS='.' read -ra versionarr <<< $(poetry version --short) - - today=$(date +'%Y%m%d%H%M%S') - - new_version="${versionarr[0]}.${versionarr[1]}.$today" - poetry version $new_version - poetry version prerelease - - NEW_TAG="v$(poetry version --short)" - echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - -# while [[ ! $(git tag -l "$tag_in_question") = '' ]] -# do -# poetry version prerelease -# tag_in_question="v$(poetry version --short)" -# loop - - #--------------------------------------------------------------- - # create build artifacts to be included as part of release - #--------------------------------------------------------------- - - name: Create build artifacts - run: | - poetry build -vvv - - - uses: ncipollo/release-action@v1 - with: - artifacts: "dist/*.gz,dist/*.whl" - artifactErrorsFailBuild: true - generateReleaseNotes: true - commit: ${{ github.ref }} - prerelease: true - tag: ${{ env.NEW_TAG }} - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish pre-release to pypi - if: github.repository_owner == 'GRIDAPPSD' - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish \ No newline at end of file diff --git a/.github/workflows/dispatch-pypi.yml b/.github/workflows/dispatch-pypi.yml deleted file mode 100644 index c0d3209..0000000 --- a/.github/workflows/dispatch-pypi.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Dispatch to PyPi - -on: - workflow_dispatch: - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - -jobs: - - publish_to_pypi: - - runs-on: ubuntu-latest - - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34b781a..0438b96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,7 @@ on: [push, pull_request] jobs: push: + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' runs-on: ubuntu-latest name: Build and push the docker container steps: @@ -19,7 +20,7 @@ jobs: - name: Log in to docker run: | if [ -n "${{ secrets.DOCKER_USERNAME }}" -a -n "${{ secrets.DOCKER_TOKEN }}" ]; then - + echo " " echo "Connecting to docker" echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin @@ -29,7 +30,7 @@ jobs: exit 1 fi fi - + - name: Build the image env: DOCKER_IMAGE_NAME: ${{ secrets.DOCKER_IMAGE_NAME }} @@ -67,11 +68,11 @@ jobs: ORG="${ORG:+${ORG}/}" IMAGE="${ORG}${{ env.DOCKER_IMAGE_NAME }}" if [ -n "${{ secrets.DOCKER_USERNAME }}" -a -n "${{ secrets.DOCKER_TOKEN }}" ]; then - + if [ -n "$TAG" -a -n "$ORG" ]; then # Get the built container name CONTAINER=`docker images --format "{{.Repository}}:{{.Tag}}" ${IMAGE}` - + echo "docker push ${CONTAINER}" docker push "${CONTAINER}" status=$? @@ -79,7 +80,7 @@ jobs: echo "Error: status $status" exit 1 fi - + echo "docker tag ${CONTAINER} ${IMAGE}:$TAG" docker tag ${CONTAINER} ${IMAGE}:$TAG status=$? @@ -87,7 +88,7 @@ jobs: echo "Error: status $status" exit 1 fi - + echo "docker push ${IMAGE}:$TAG" docker push ${IMAGE}:$TAG status=$? @@ -96,5 +97,5 @@ jobs: exit 1 fi fi - + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1618a0..dcdd084 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ on: publish-to: description: "Publish to pypi or pypi-test" required: true - type: string + type: choice default: "pypi" options: - "pypi" @@ -96,7 +96,6 @@ jobs: ./scripts/poetry_build.sh - name: Push artifacts to github - # if: inputs.publish-to == 'pypi' uses: ncipollo/release-action@v1 with: artifacts: "dist/*.gz,dist/*.whl" @@ -140,9 +139,9 @@ jobs: # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS='--repository testpypi'" >> $GITHUB_ENV cd gridappsd-python-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish cd ../gridappsd-field-bus-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish \ No newline at end of file diff --git a/.github/workflows/release_wheel.yml b/.github/workflows/release_wheel.yml deleted file mode 100644 index c6ac80a..0000000 --- a/.github/workflows/release_wheel.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Publish Python package -on: - push: - tags: - - 'v*.*.*' -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" \ No newline at end of file diff --git a/.gitignore b/.gitignore index fda0577..751a919 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,7 @@ ENV/ # visual studio code .vscode/ + +# pytest cache +.pytest_cache + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..94c3e96 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-xml + - id: forbid-new-submodules + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-merge-conflict + - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty + - id: pretty-format-json + + +#- repo: https://github.com/pre-commit/mirrors-autopep8 +# rev: v1.6.0 +# hooks: +# - id: autopep8 + +- repo: https://github.com/craig8/mirrors-yapf + rev: b84f670025671a341d0afd2b06b877b195d65c0f # Use the sha / tag you want to point at + hooks: + - id: yapf + name: yapf + description: "A formatter for Python files." + entry: yapf + language: python + types: [ python ] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index 7305a71..635e419 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,5 @@ def test_gridappsd_status(gridappsd_client): gappsd.set_service_status("Foo") assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - ``` diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-field-bus-lib/CHANGELOG.md b/gridappsd-field-bus-lib/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/README.md b/gridappsd-field-bus-lib/README.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/gridappsd/__no_init__here b/gridappsd-field-bus-lib/gridappsd/__no_init__here new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd/field_interface/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py similarity index 64% rename from gridappsd/field_interface/__init__.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py index 2181da7..66eba6b 100644 --- a/gridappsd/field_interface/__init__.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py @@ -1,9 +1,9 @@ from typing import List -from gridappsd.field_interface.context import ContextManager +from gridappsd.field_interface.context import LocalContext from gridappsd.field_interface.interfaces import MessageBusDefinition __all__: List[str] = [ - "ContextManager", + "LocalContext", "MessageBusDefinition" ] diff --git a/gridappsd/field_interface/agents/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py similarity index 100% rename from gridappsd/field_interface/agents/__init__.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py new file mode 100644 index 0000000..ff48e57 --- /dev/null +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py @@ -0,0 +1,353 @@ +import dataclasses +import importlib +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime +from typing import Dict + +from cimgraph.loaders import ConnectionParameters, gridappsd +from cimgraph.loaders.gridappsd import GridappsdConnection +from cimgraph.models import DistributedModel, SecondaryArea, SwitchArea + +import gridappsd.topics as t +from gridappsd.field_interface.context import LocalContext +from gridappsd.field_interface.gridappsd_field_bus import GridAPPSDMessageBus +from gridappsd.field_interface.interfaces import (FieldMessageBus, + MessageBusDefinition) + +cim = None +sparql = None + +_log = logging.getLogger(__name__) + + +def set_cim_profile(cim_profile): + global cim + cim = importlib.import_module('cimgraph.data_profile.' + cim_profile) + gridappsd.set_cim_profile(cim_profile) + + +@dataclass +class AgentRegistrationDetails: + agent_id: str + app_id: str + description: str + upstream_message_bus_id: FieldMessageBus.id + downstream_message_bus_id: FieldMessageBus.id + + +class DistributedAgent: + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config, + agent_area_dict=None, + simulation_id=None, + cim_profile: str = None): + """ + Creates a DistributedAgent object that connects to the specified message + buses and gets context based on feeder id and area id. + """ + _log.debug(f"Creating DistributedAgent: {self.__class__.__name__}") + self.upstream_message_bus = None + self.downstream_message_bus = None + self.simulation_id = simulation_id + self.context = None + + #TODO: Change params and connection to local connection + self.params = ConnectionParameters() + self.connection = GridappsdConnection(self.params) + + self.app_id = agent_config['app_id'] + self.description = agent_config['description'] + dt = datetime.now() + ts = datetime.timestamp(dt) + self.agent_id = "da_" + self.app_id + "_" + str(int(ts)) + self.agent_area_dict = agent_area_dict + + if upstream_message_bus_def is not None: + if upstream_message_bus_def.is_ot_bus: + self.upstream_message_bus = GridAPPSDMessageBus( + upstream_message_bus_def) + # else: + # self.upstream_message_bus = VolttronMessageBus(upstream_message_bus_def) + + if downstream_message_bus_def is not None: + if downstream_message_bus_def.is_ot_bus: + self.downstream_message_bus = GridAPPSDMessageBus( + downstream_message_bus_def) + # else: + # self.downstream_message_bus = VolttronMessageBus(downstream_message_bus_def) + + # self.context = ContextManager.get(self.feeder_id, self.area_id) + + #if agent_dict is not None: + # self.addressable_equipments = agent_dict['addressable_equipment'] + # self.unaddressable_equipments = agent_dict['unaddressable_equipment'] + + def connect(self): + + if self.upstream_message_bus is not None: + self.upstream_message_bus.connect() + if self.downstream_message_bus is not None: + self.downstream_message_bus.connect() + if self.downstream_message_bus is None and self.upstream_message_bus is None: + raise ValueError( + "Either upstream or downstream bus must be specified!") + + if self.agent_area_dict is None: + context = LocalContext.get_context_by_message_bus( + self.downstream_message_bus) + self.agent_area_dict = context['data'] + + self.subscribe_to_measurement() + self.subscribe_to_messages() + self.subscribe_to_requests() + + if ('context_manager' not in self.app_id): + LocalContext.register_agent(self.downstream_message_bus, + self.upstream_message_bus, self) + + def subscribe_to_measurement(self): + if self.simulation_id is None: + self.downstream_message_bus.subscribe( + t.field_output_topic(self.downstream_message_bus.id), + self.on_measurement) + else: + topic = t.field_output_topic(self.downstream_message_bus.id, + self.simulation_id) + _log.debug(f"subscribing to simulation output on topic {topic}") + self.downstream_message_bus.subscribe(topic, + self.on_simulation_output) + + def subscribe_to_messages(self): + + self.downstream_message_bus.subscribe( + t.field_message_bus_topic(self.downstream_message_bus), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_topic(self.upstream_message_bus), + self.on_upstream_message) + + _log.debug( + f"Subscribing to messages on application topics: \n {t.field_message_bus_app_topic(self.downstream_message_bus.id, self.app_id)} \ + \n {t.field_message_bus_app_topic(self.upstream_message_bus.id, self.app_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_message_bus_app_topic(self.downstream_message_bus.id, + self.app_id), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_app_topic(self.upstream_message_bus.id, + self.app_id), + self.on_upstream_message) + + _log.debug( + f"Subscribing to message on agents topics: \n {t.field_message_bus_agent_topic(self.downstream_message_bus.id, self.agent_id)} \ + \n {t.field_message_bus_agent_topic(self.upstream_message_bus.id, self.agent_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_message_bus_agent_topic(self.downstream_message_bus.id, + self.agent_id), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_agent_topic(self.upstream_message_bus.id, + self.agent_id), + self.on_upstream_message) + + def subscribe_to_requests(self): + + _log.debug( + f"Subscribing to requests on agents queue: \n {t.field_agent_request_queue(self.downstream_message_bus.id, self.agent_id)} \ + \n {t.field_agent_request_queue(self.upstream_message_bus.id, self.agent_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_agent_request_queue(self.downstream_message_bus.id, + self.agent_id), + self.on_request_from_downstream) + self.downstream_message_bus.subscribe( + t.field_agent_request_queue(self.upstream_message_bus.id, + self.agent_id), + self.on_request_from_uptream) + + def on_measurement(self, headers: Dict, message) -> None: + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def on_simulation_output(self, headers, message): + self.on_measurement(headers=headers, message=message) + + def on_upstream_message(self, headers: Dict, message) -> None: + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def on_downstream_message(self, headers: Dict, message) -> None: + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def on_request_from_uptream(self, headers: Dict, message): + self.on_request(self.upstream_message_bus, headers, message) + + def on_request_from_downstream(self, headers: Dict, message): + self.on_request(self.downstream_message_bus, headers, message) + + def on_request(self, message_bus, headers: Dict, message): + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def get_registration_details(self): + details = AgentRegistrationDetails(str(self.agent_id), self.app_id, + self.description, + self.upstream_message_bus.id, + self.downstream_message_bus.id) + return dataclasses.asdict(details) + + +''' TODO this has not been implemented yet, so we are commented them out for now. + # not all agent would use this + def on_control(self, control): + device_id = control.get('device') + command = control.get('command') + self.control_device(device_id, command) + + def control_device(self, device_id, command): + device_topic = self.devices.get(device_id) + self.secondary_message_bus.publish(device_topic, command)''' + + +class FeederAgent(DistributedAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + feeder_dict=None, + simulation_id=None): + super(FeederAgent, + self).__init__(upstream_message_bus_def, + downstream_message_bus_def, agent_config, + feeder_dict, simulation_id) + self.feeder_area = None + self.downstream_message_bus_def = downstream_message_bus_def + if self.agent_area_dict is not None: + feeder = cim.Feeder(mRID=self.downstream_message_bus_def.id) + self.feeder_area = DistributedModel(connection=self.connection, + feeder=feeder, + topology=self.agent_area_dict) + + + def connect(self): + super().connect() + if self.feeder_area is None: + feeder = cim.Feeder(mRID=self.downstream_message_bus_def.id) + self.feeder_area = DistributedModel(connection=self.connection, + feeder=feeder, + topology=self.agent_area_dict) + + +class SwitchAreaAgent(DistributedAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + switch_area_dict=None, + simulation_id=None): + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, switch_area_dict, simulation_id) + self.switch_area = None + self.downstream_message_bus_def = downstream_message_bus_def + if self.agent_area_dict is not None: + self.switch_area = SwitchArea(self.downstream_message_bus_def.id, + self.connection) + self.switch_area.initialize_switch_area(self.agent_area_dict) + + + def connect(self): + super().connect() + if self.switch_area is None: + self.switch_area = SwitchArea(self.downstream_message_bus_def.id, + self.connection) + self.switch_area.initialize_switch_area(self.agent_area_dict) + + +class SecondaryAreaAgent(DistributedAgent): + + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, + downstream_message_bus_def: MessageBusDefinition, + agent_config: Dict, + secondary_area_dict=None, + simulation_id=None): + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, secondary_area_dict, simulation_id) + self.secondary_area = None + self.downstream_message_bus_def = downstream_message_bus_def + if self.agent_area_dict is not None: + self.secondary_area = SecondaryArea(self.downstream_message_bus_def.id, + self.connection) + self.secondary_area.initialize_secondary_area(self.agent_area_dict) + + + def connect(self): + super().connect() + if self.secondary_area is None: + self.secondary_area = SecondaryArea(self.downstream_message_bus_def.id, + self.connection) + self.secondary_area.initialize_secondary_area(self.agent_area_dict) + + +class CoordinatingAgent: + """ + A CoordinatingAgent performs following functions: + 1. Spawns distributed agents + 2. Publishes compiled output to centralized OT bus + 3. Distributes centralized output to Feeder bus and distributed agents + 4. May have connected devices and control those devices + + upstream, peer , downstream and broadcast + """ + + def __init__(self, + feeder_id, + system_message_bus_def: MessageBusDefinition, + simulation_id=None): + self.feeder_id = feeder_id + self.distributed_agents = [] + + self.system_message_bus = GridAPPSDMessageBus(system_message_bus_def) + self.system_message_bus.connect() + + #This will change when we have multiple feeders per system + self.downstream_message_bus = self.system_message_bus + + # self.context = ContextManager.getContextByFeeder(self.feeder_id) + # print(self.context) + # self.addressable_equipments = self.context['data']['addressable_equipment'] + # self.unaddressable_equipments = self.context['data']['unaddressable_equipment'] + # self.switch_areas = self.context['data']['switch_areas'] + + # self.subscribe_to_feeder_bus() + + def spawn_distributed_agent(self, distributed_agent: DistributedAgent): + distributed_agent.connect() + self.distributed_agents.append(distributed_agent) + + +''' + def on_control(self, control): + device_id = control.get('device') + command = control.get('command') + self.control_device(device_id, command) + + def publish_to_distribution_bus(self,message): + self.publish_to_downstream_bus(message) + + def publish_to_distribution_bus_agent(self,agent_id, message): + self.publish_to_downstream_bus_agent(agent_id, message) + + def control_device(self, device_id, command): + device_topic = self.devices.get(device_id) + self.secondary_message_bus.publish(device_topic, command)''' diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py new file mode 100644 index 0000000..35dec86 --- /dev/null +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py @@ -0,0 +1,51 @@ +from gridappsd.field_interface.interfaces import FieldMessageBus +import dataclasses +import gridappsd.topics as t +import json + + + +class LocalContext: + + @classmethod + def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_mrid, area_id=None): + + request = {'request_type' : 'get_context', + 'modelId': feeder_mrid, + 'areaId': area_id} + response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) + return response + + @classmethod + def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): + """ + return agents/devices based on downstream message bus as input + + """ + request = {'request_type' : 'get_context', + 'downstream_message_bus_id': downstream_message_bus.id + } + return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) + + @classmethod + def register_agent(cls, downstream_message_bus: FieldMessageBus, upstream_message_bus: FieldMessageBus, agent): + """ + Sends the newly created distributed agent's info to OT bus + + """ + request = {'request_type' : 'register_agent', + 'agent' : agent.get_registration_details()} + downstream_message_bus.send(t.context_request_queue(downstream_message_bus.id), request) + upstream_message_bus.send(t.context_request_queue(upstream_message_bus.id), request) + + @classmethod + def get_agents(cls, downstream_message_bus: FieldMessageBus): + """ + Sends the newly created distributed agent's info to OT bus + + """ + request = {'request_type' : 'get_agents'} + return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) + +# Provide context based on router (ip trace) or PKI +# Maybe able to emulate/simulate diff --git a/gridappsd/field_interface/gridappsd_field_bus.py b/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py similarity index 68% rename from gridappsd/field_interface/gridappsd_field_bus.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py index 7d93f58..8cb1fa5 100644 --- a/gridappsd/field_interface/gridappsd_field_bus.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py @@ -1,9 +1,10 @@ -from gridappsd.field_interface.interfaces import MessageBusDefinition -from gridappsd.field_interface.interfaces import FeederMessageBus from gridappsd import GridAPPSD +from gridappsd.field_interface.interfaces import FieldMessageBus +from gridappsd.field_interface.interfaces import MessageBusDefinition +from typing import Any -class GridAPPSDMessageBus(FeederMessageBus): +class GridAPPSDMessageBus(FieldMessageBus): def __init__(self, definition: MessageBusDefinition): super(GridAPPSDMessageBus, self).__init__(definition) self._id = definition.id @@ -36,12 +37,20 @@ def subscribe(self, topic, callback): def unsubscribe(self, topic): pass - def publish(self, data, topic: str = None): + def send(self, topic: str, message: Any): """ Publish device specific data to the concrete message bus. """ - pass - + if self.gridappsd_obj is not None: + self.gridappsd_obj.send(topic, message) + + def get_response(self, topic, message, timeout=5): + """ + Sends a message on a specific concrete queue, waits and returns the response + """ + if self.gridappsd_obj is not None: + return self.gridappsd_obj.get_response(topic, message, timeout) + def disconnect(self): """ Disconnect the device from the concrete message bus. diff --git a/gridappsd/field_interface/interfaces.py b/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py similarity index 63% rename from gridappsd/field_interface/interfaces.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py index 3b776a9..8e42e66 100644 --- a/gridappsd/field_interface/interfaces.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum +import gridappsd.topics as t import logging from os import PathLike from pathlib import Path @@ -164,194 +165,24 @@ def unsubscribe(self, topic): pass @abstractmethod - def publish(self, data, topic: str = None): + def send(self, topic, message): """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. - """ - pass - - -class SwitchAreaMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - - @abstractmethod - def connect(self): - """ - Connect to the concrete message bus that implements this interface. - """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): - """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. - """ - pass - - -class SecondaryMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - - @abstractmethod - def connect(self): - """ - Connect to the concrete message bus that implements this interface. - """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): - """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. + Publish device specific message to the concrete message bus. """ pass - -class FeederMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - @abstractmethod - def connect(self): + def get_response(self, topic, message, timeout): """ - Connect to the concrete message bus that implements this interface. + Sends a message on a specific queue, waits and returns the response """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): + + def get_agent_response(self, agent_id, message, timeout): """ - Publish device specific data to the concrete message bus. + Sends a message on a specific agent's request queue, waits and returns the response """ - pass + topic = "{}.request.{}.{}".format(t.BASE_FIELD_QUEUE,self.id, agent_id) + self.get_response(topic, message, timeout) @abstractmethod def disconnect(self): @@ -361,8 +192,6 @@ def disconnect(self): pass - - class MessageBusDefinitions: def __init__( self, diff --git a/gridappsd-field-bus-lib/info/CHANGELOG.md b/gridappsd-field-bus-lib/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/info/VERSION b/gridappsd-field-bus-lib/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/gridappsd-field-bus-lib/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-field-bus-lib/info/requirements.txt b/gridappsd-field-bus-lib/info/requirements.txt new file mode 100644 index 0000000..2e37a34 --- /dev/null +++ b/gridappsd-field-bus-lib/info/requirements.txt @@ -0,0 +1,22 @@ +certifi==2023.5.7 ; python_full_version >= "3.7.9" and python_version < "4" +charset-normalizer==3.1.0 ; python_full_version >= "3.7.9" and python_version < "4" +cim-graph==0.1.20230508194360a0 ; python_full_version >= "3.7.9" and python_version < "4.0" +dateutils==0.6.12 ; python_full_version >= "3.7.9" and python_version < "4.0" +docopt==0.6.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +gridappsd-python==2023.5.2a0; python_full_version >= "3.7.9" and python_version < "4.0" +idna==3.4 ; python_full_version >= "3.7.9" and python_version < "4" +importlib-metadata==4.13.0 ; python_full_version >= "3.7.9" and python_version < "3.8" +isodate==0.6.1 ; python_full_version >= "3.7.9" and python_version < "4.0" +pyparsing==3.0.9 ; python_full_version >= "3.7.9" and python_version < "4.0" +python-dateutil==2.8.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +pytz==2022.7.1 ; python_full_version >= "3.7.9" and python_version < "4.0" +pyyaml==6.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +rdflib==6.3.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +requests==2.28.2 ; python_full_version >= "3.7.9" and python_version < "4" +six==1.16.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +sparqlwrapper==2.0.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +stomp-py==6.0.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_full_version >= "3.7.9" and python_version < "3.8" +urllib3==1.26.15 ; python_full_version >= "3.7.9" and python_version < "4" +xsdata==22.12 ; python_full_version >= "3.7.9" and python_version < "4.0" +zipp==3.15.0 ; python_full_version >= "3.7.9" and python_version < "3.8" diff --git a/gridappsd-field-bus-lib/poetry.toml b/gridappsd-field-bus-lib/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/gridappsd-field-bus-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml new file mode 100644 index 0000000..6da09ad --- /dev/null +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -0,0 +1,39 @@ +[tool.poetry] +name = "gridappsd-field-bus" +version = "2023.5.2a0" +description = "GridAPPS-D Field Bus Implementation" +authors = [ + "C. Allwardt <3979063+craig8@users.noreply.github.com>", + "P. Sharma =1.2.0"] +build-backend = "poetry.core.masonry.api" diff --git a/gridappsd-python-lib/CHANGELOG.md b/gridappsd-python-lib/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-python-lib/README.md b/gridappsd-python-lib/README.md new file mode 100644 index 0000000..635e419 --- /dev/null +++ b/gridappsd-python-lib/README.md @@ -0,0 +1,328 @@ +[![Run All Pytests](https://github.com/GRIDAPPSD/gridappsd-python/actions/workflows/run-pytest.yml/badge.svg)](https://github.com/GRIDAPPSD/gridappsd-python/actions/workflows/run-pytest.yml) + +# gridappsd-python +Python library for developing applications and services against the gridappsd api + +## Requirements + +The gridappsd-python library requires a python version >= 3.6 and < 4 in order to work properly (Note no testing +has been done with python 4 to date). + +## Installation + +The recommended installation of `gridappsd-python` is in a separate virtual environment. Executing the following +will create an environment called `griddapps-env`. + +```shell +python3 -m venv gridappsd-env +``` + +Sourcing the gridappsd-env activates the newly created python environment. + +```shell +source gridappsd-env/bin/activate +``` + +Upgrade pip to the latest (some packages require 19.0+ version of pip). + +```shell +python -m pip install pip --upgrade +``` + +Install the latest `gridappsd-python` and its dependencies in the virtual environment. + +```shell +pip install gridappsd-python +``` + +### Verifying things are working properly + +The following code snippet assumes you have created a gridappsd instance using the steps in +https://github.com/GRIDAPPSD/gridappsd-docker. + +Create a test script (tester.py) with the following content. + +```python + +from gridappsd import GridAPPSD + +def on_message_callback(header, message): + print(f"header: {header} message: {message}") + +# Note these should be changed on the server in a cyber secure environment! +username = "app_user" +password = "1234App" + +# Note: there are other parameters for connecting to +# systems other than localhost +gapps = GridAPPSD(username=username, password=password) + +assert gapps.connected + +gapps.send('send.topic', {"foo": "bar"}) + +# Note we are sending the function not executing the function in the second parameter +gapps.subscribe('subscribe.topic', on_message_callback) + +gapps.send('subcribe.topic', 'A message about subscription') + +time.sleep(5) + +gapps.close() + +``` + +Start up the gridappsd-docker enabled platform. Then run the following to execute the tester.py script + +```shell +python tester.py +``` + +## Application Developers + +### Deployment + +Please see [DOCKER_CONTAINER.md](DOCKER_CONTAINER.md) for working within the docker application base container. + +### Local Development + +Developing applications against gridappsd using the `gridappsd-python` library should follow the same steps +as above, however with a couple of environmental variables specified. The following environmental variables are +available to provide the same context that would be available from inside the application docker container. These +are useful to know for developing your application outside of the docker context (e.g. in a python notebook). + +***NOTE: you can also define these your ~./bashrc file so you don't have to specify them all the time*** + +```shell +# export allows all processes started by this shell to have access to the global variable + +# address where the gridappsd server is running - default localhost +export GRIDAPPSD_ADDRESS=localhost + +# port to connect to on the gridappsd server (the stomp client port) +export GRIDAPPSD_PORT=61613 + +# username to connect to the gridappsd server +export GRIDAPPSD_USER=app_user + +# password to connect to the gridappsd server +export GRIDAPPSD_PASSWORD=1234App + +# Note these should be changed on the server in a cyber secure environment! +``` + +The following is the same tester code as above, but with the environment variables set. The environment variables +should be set in your environment when running the application inside our docker container. + +```python + +from gridappsd import GridAPPSD + +def on_message_callback(header, message): + print(f"header: {header} message: {message}") + +# Create GridAPPSD object and connect to the gridappsd server. +gapps = GridAPPSD() + +assert gapps.connected + +gapps.send('send.topic', {"foo": "bar"}) + +# Note we are sending the function not executing the function in the second parameter +gapps.subscribe('subscribe.topic', on_message_callback) + +gapps.send('subcribe.topic', 'A message about subscription') + +time.sleep(5) + +gapps.close() + +``` + +## Developers + +This project uses poetry to build the environment for execution. Follow the instructions +https://python-poetry.org/docs/#installation to install poetry. As a developer I prefer not to have poetry installed +in the same virtual environment that my projects are in. + +Clone the github repository: + +```shell +git clone https://github.com/GRIDAPPSD/gridappsd-python -b develop +cd gridappsd-python +``` + +The following commands build and install a local wheel into an environment created just for this package. + +```shell +# Build the project (stores in dist directory both .tar.gz and .whl file) +poetry build + +# Install the wheel into the environment and the dev dependencies +poetry install + +# Install only the library dependencies +poetry install --no-dev +``` + +***Note:*** Poetry does not have a setup.py that you can install in editable mode like with pip install -e ., however +you can extract the generated setup.py file from the built tar.gz file in the dist directory. Just extract the +.tar.gz file and copy the setup.py file from the extracted directory to the root of gridappsd-python. Then you can +enable editing through pip install -e. as normal. + + +## Testing + +Testing has become an integral part of the software lifecycle. The `gridappsd-python` library has both unit and +integration tests available to be run. In order to execute these, you must have installed the gridappsd-python library +as above with dev-dependencies. + +During the testing phase the docker containers required for the tests are downloaded from +dockerhub and started. By default the `develop` tag is used to test the library using pytest. +One can customize the docker image tag by setting the environmental +variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing +pytest with the following: + +```shell script + +# Export environmental variables and all tests will use the same tag (other_tag) to pull from docker hub. +# Default tag is develop +export GRIDAPPSD_TAG_ENV=other_tag +pytest + +# Tests also require the username and password to be avaialable as environmental variables +# in order for them to properly run these tests +export GRIDAPPSD_USER=user +export GRIDAPPSD_PASSWORD=pass + +pytest +``` + + ***NOTE: the first running the tests will download all of the docker images associated with the + [GOSS-GridAPPS-D](http://github.com/GRIDAPPSD/GOSS-GridAPPS-D) repository. This process may take some time.*** + +### Running tests created in a new project + +The `gridappsd-python` library exposes a testing environment through the `gridappsd.docker_handler` module. Including the following +`conftest.py` in the root of your base test directory allows tests to reference these. Using these fixtures will start all of the +base containers required for `gridappsd` to run. + +```python + +# conftest.py +# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. +# +# Tested project structure an layout +# +# project-folder\ +# mainmodule\ +# __init__.py +# myapplication.py +# tests\ +# conftest.py +# test_myapplication.py +# README.md + +import logging +import os +import sys + +import pytest + +from gridappsd import GridAPPSD, GOSS +from gridappsd.docker_handler import run_dependency_containers, run_gridappsd_container, Containers + +levels = dict( + CRITICAL=50, + FATAL=50, + ERROR=40, + WARNING=30, + WARN=30, + INFO=20, + DEBUG=10, + NOTSET=0 +) + +# Get string representation of the log level passed +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") + +# Make sure the level passed is one of the valid levels. +if LOG_LEVEL not in levels.keys(): + raise AttributeError("Invalid LOG_LEVEL environmental variable set.") + +# Set the numeric version of log level to pass to the basicConfig function +LOG_LEVEL = levels[LOG_LEVEL] + +logging.basicConfig(stream=sys.stdout, level=LOG_LEVEL, + format="%(asctime)s|%(levelname)s|%(name)s|%(message)s") +logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +logging.getLogger("docker.utils.config").setLevel(logging.INFO) +logging.getLogger("docker.auth").setLevel(logging.INFO) + + +STOP_CONTAINER_AFTER_TEST = os.environ.get('GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', True) + + +@pytest.fixture(scope="module") +def docker_dependencies(): + print("Docker dependencies") + # Containers.reset_all_containers() + + with run_dependency_containers(stop_after=STOP_CONTAINER_AFTER_TEST) as dep: + yield dep + print("Cleanup docker dependencies") + + +@pytest.fixture +def gridappsd_client(request, docker_dependencies): + with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + gappsd = GridAPPSD() + gappsd.connect() + assert gappsd.connected + models = gappsd.query_model_names() + assert models is not None + if request.cls is not None: + request.cls.gridappsd_client = gappsd + yield gappsd + + gappsd.disconnect() + + +@pytest.fixture +def goss_client(docker_dependencies): + with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + goss = GOSS() + goss.connect() + assert goss.connected + + yield goss + +``` + +Using the above fixtures from inside a test module and test function looks like the following: + +```python + +# Example test function using the gridappsd_client fixture + +@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) +def test_gridappsd_status(gridappsd_client): + gappsd = gridappsd_client + assert "helics_goss_bridge.py" == gappsd.get_application_id() + assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value + assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value + gappsd.set_application_status("RUNNING") + + assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value + assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value + + gappsd.set_service_status("COMPLETE") + assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value + + # Invalid + gappsd.set_service_status("Foo") + assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value +``` + diff --git a/gridappsd/conf/entrypoint.sh b/gridappsd-python-lib/conf/entrypoint.sh similarity index 100% rename from gridappsd/conf/entrypoint.sh rename to gridappsd-python-lib/conf/entrypoint.sh diff --git a/gridappsd/conf/run-gridappsd.sh b/gridappsd-python-lib/conf/run-gridappsd.sh similarity index 100% rename from gridappsd/conf/run-gridappsd.sh rename to gridappsd-python-lib/conf/run-gridappsd.sh diff --git a/gridappsd/__init__.py b/gridappsd-python-lib/gridappsd/__init__.py similarity index 100% rename from gridappsd/__init__.py rename to gridappsd-python-lib/gridappsd/__init__.py diff --git a/gridappsd/__main__.py b/gridappsd-python-lib/gridappsd/__main__.py similarity index 100% rename from gridappsd/__main__.py rename to gridappsd-python-lib/gridappsd/__main__.py diff --git a/gridappsd/app_registration.py b/gridappsd-python-lib/gridappsd/app_registration.py similarity index 100% rename from gridappsd/app_registration.py rename to gridappsd-python-lib/gridappsd/app_registration.py diff --git a/gridappsd/difference_builder.py b/gridappsd-python-lib/gridappsd/difference_builder.py similarity index 100% rename from gridappsd/difference_builder.py rename to gridappsd-python-lib/gridappsd/difference_builder.py diff --git a/gridappsd-python-lib/gridappsd/docker_handler.py b/gridappsd-python-lib/gridappsd/docker_handler.py new file mode 100644 index 0000000..29bf751 --- /dev/null +++ b/gridappsd-python-lib/gridappsd/docker_handler.py @@ -0,0 +1,723 @@ +# #!/usr/bin/env python3 + +# import contextlib +# import logging +# import os +# import random +# import re +# import shutil +# import tarfile +# import threading +# import time +# import urllib.request +# from copy import deepcopy +# from datetime import datetime, timedelta +# from pathlib import Path +# from pprint import pformat +# from subprocess import PIPE +# from typing import Optional, Union + +# import stomp + +# from gridappsd import GridAPPSD +# from gridappsd.goss import GRIDAPPSD_ENV_ENUM + +# logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +# logging.getLogger("docker.auth").setLevel(logging.INFO) +# logging.getLogger("docker.utils").setLevel(logging.INFO) + +# _log = logging.getLogger("gridappsd.docker_handler") + +# try: +# from python_on_whales import docker +# HAS_DOCKER = True +# except ImportError: +# _log.warning( +# "Docker api not loaded. pip install docker to install as package.") +# HAS_DOCKER = False + +# if HAS_DOCKER: + +# # The following variable is used for creating a volume for the gridappsd container +# # to utilize. It allows the ability to use multiple containers to run tests +# # along side each other. +# GRIDAPPSD_CONFIG_VOLUME_NAME = f"gridappsd_config_{random.randint(1,100)}" + +# # This named container will be used to hold configuration/folders so that other containers +# # that start can use them. To use add "volume_from": [CONFIGURATION_CONTAINER_NAME] and +# # the mount point within the container will also be within the service container. +# CONFIGURATION_CONTAINER_NAME = "testconfig" + +# def expand_all(user_path): +# return os.path.expandvars(os.path.expanduser(user_path)) + +# __TMP_ROOT__ = "/tmp/assets" +# if Path(__TMP_ROOT__).exists(): +# shutil.rmtree(__TMP_ROOT__, ignore_errors=True) +# os.makedirs(__TMP_ROOT__) + +# # This path needs to be the path to the repo where configuration files are located. +# GRIDAPPSD_CONF_DIR = Path(__file__).parent.parent.joinpath("conf") + +# assert Path(GRIDAPPSD_CONF_DIR).joinpath("entrypoint.sh").exists() +# assert Path(GRIDAPPSD_CONF_DIR).joinpath("run-gridappsd.sh").exists() + +# GRIDAPPSD_DATA_REPO = str(Path(__TMP_ROOT__).joinpath("mysql").resolve()) + +# os.makedirs(GRIDAPPSD_DATA_REPO, exist_ok=True) +# if not Path(GRIDAPPSD_DATA_REPO).is_dir(): +# raise AttributeError( +# f"Invalid GRIDAPPSD_DATA_REPO couldn't make or doesn't exist {GRIDAPPSD_DATA_REPO}" +# ) + +# MYSQL_SCHEMA_INIT_DIR = f'{GRIDAPPSD_DATA_REPO}/docker-entrypoint-initdb.d' + +# def mysql_setup(): +# """ +# Downloads gridappsd_mysql_dump.sql into the MYSQL_SCHEMA_INIT_DIR init directory. +# This will then be mounted into the mysql container. +# """ +# # Downlaod mysql file +# _log.debug("Downloading mysql data file from Bootstrap repository") +# mysql_file = f'{MYSQL_SCHEMA_INIT_DIR}/gridappsd_mysql_dump.sql' +# if os.path.isdir(mysql_file): +# raise RuntimeError( +# f"mysql datafile is directory, delete {mysql_file} using sudo rm -rf {mysql_file}" +# ) +# if not os.path.isdir(MYSQL_SCHEMA_INIT_DIR): +# os.makedirs(MYSQL_SCHEMA_INIT_DIR, 0o0775, exist_ok=True) +# urllib.request.urlretrieve( +# 'https://raw.githubusercontent.com/GRIDAPPSD/Bootstrap/master/gridappsd_mysql_dump.sql', +# filename=mysql_file) + +# # Modify the mysql file to allow connections from gridappsd container +# with open(mysql_file, "r") as sources: +# lines = sources.readlines() +# with open(mysql_file, "w") as sources: +# for line in lines: +# sources.write(re.sub(r'localhost', '%', line)) +# assert Path(mysql_file).exists() + +# # Use the environmental variable if specified otherwise use the develop tag. +# DEFAULT_GRIDAPPSD_TAG = os.environ.get('GRIDAPPSD_TAG_ENV', 'develop') + +# # Network to connect all of the containers up to by default. +# NETWORK = "test_my_network" + +# __TPL_DEPENDENCY_CONFIG__ = { +# "influxdb": { +# "start": True, +# "image": "gridappsd/influxdb:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8086/tcp": 8086 +# }, +# "environment": { +# "INFLUXDB_DB": "proven" +# } +# }, +# "redis": { +# "start": True, +# "image": "redis:3.2.11-alpine", +# "pull": True, +# "ports": { +# "6379/tcp": 6379 +# }, +# "environment": [], +# "entrypoint": "redis-server --appendonly yes", +# }, +# "blazegraph": { +# "start": True, +# "image": "gridappsd/blazegraph:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8080/tcp": 8889 +# }, +# "environment": [] +# }, +# "mysql": { +# "start": +# True, +# "image": +# "mysql/mysql-server:5.7", +# "pull": +# True, +# "ports": { +# "3306/tcp": 3306 +# }, +# "environment": { +# "MYSQL_RANDOM_ROOT_PASSWORD": "yes", +# "MYSQL_PORT": "3306" +# }, +# # "volumes": { +# # "/home/gridappsd/test-assets": {"bind": "/whatthehell", "mode": "rw"} +# # #"test-assets": {"bind": "/whatthehell", "mode": "rw"} +# # #data_dir + "/dumps/": {"bind": "/docker-entrypoint-initdb.d/", "mode": "ro"} +# # }, +# # Our own so we can create +# "volumes_required": [ +# dict(name="mysql_config", +# local_path=MYSQL_SCHEMA_INIT_DIR, +# container_path="/docker-entrypoint-initdb.d") +# ], +# "volumes_from": ["mysql_config"], +# "onsetupfn": +# mysql_setup +# }, +# "proven": { +# "start": True, +# "image": "gridappsd/proven:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8080/tcp": 18080 +# }, +# "environment": { +# "PROVEN_SERVICES_PORT": "18080", +# "PROVEN_SWAGGER_HOST_PORT": "localhost:18080", +# "PROVEN_USE_IDB": "true", +# "PROVEN_IDB_URL": "http://influxdb:8086", +# "PROVEN_IDB_DB": "proven", +# "PROVEN_IDB_RP": "autogen", +# "PROVEN_IDB_USERNAME": "root", +# "PROVEN_IDB_PASSWORD": "root", +# "PROVEN_T3DIR": "/proven" +# }, +# "links": { +# "influxdb": "influxdb" +# } +# } +# } + +# __TPL_GRIDAPPSD_CONFIG__ = { +# "gridappsd": { +# "start": +# True, +# "image": +# "gridappsd/gridappsd:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": +# True, +# "ports": { +# "61613/tcp": 61613, +# "61614/tcp": 61614, +# "61616/tcp": 61616 +# }, +# "environment": { +# "PATH": +# "/gridappsd/bin:/gridappsd/lib:/gridappsd/services/fncsgossbridge/service:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin", +# "DEBUG": 1, +# "START": 1 +# }, +# "links": { +# "mysql": "mysql", +# "influxdb": "influxdb", +# "blazegraph": "blazegraph", +# "proven": "proven", +# "redis": "redis" +# }, +# "volumes_required": [ +# dict(name=GRIDAPPSD_CONFIG_VOLUME_NAME, +# local_path=GRIDAPPSD_CONF_DIR, +# container_path="/startup/conf") +# ], +# "volumes_from": [GRIDAPPSD_CONFIG_VOLUME_NAME], +# "entrypoint": +# "/startup/conf/entrypoint.sh", +# "command": +# "/startup/conf/entrypoint.sh" +# } +# } + +# def __update_template_data__(data, update_dict): +# data_cpy = deepcopy(data) +# for k, v in data_cpy.items(): +# for u, p in update_dict.items(): +# v['image'] = v['image'].replace(u, p) + +# return data_cpy + +# __replace_dict__ = {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG} +# DEFAULT_DOCKER_DEPENDENCY_CONFIG = __update_template_data__( +# __TPL_DEPENDENCY_CONFIG__, __replace_dict__) +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG = __update_template_data__( +# __TPL_GRIDAPPSD_CONFIG__, __replace_dict__) + +# def update_gridappsd_tag(new_gridappsd_tag): +# """ +# Update the default tag used within the dependency and gridappsd containers to be +# what is specified in the new gridappsd_tag variable +# """ +# global DEFAULT_GRIDAPPSD_TAG + +# DEFAULT_GRIDAPPSD_TAG = new_gridappsd_tag +# _log.info(f"Updated gridappsd docker tag {DEFAULT_GRIDAPPSD_TAG} ") +# __replace_dict__.update( +# {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG}) +# DEFAULT_DOCKER_DEPENDENCY_CONFIG.update( +# __update_template_data__(__TPL_DEPENDENCY_CONFIG__, +# __replace_dict__)) +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG.update( +# __update_template_data__(__TPL_GRIDAPPSD_CONFIG__, +# __replace_dict__)) + +# class Containers: +# """ +# This class allows the creation/management of containers created by the gridappsd +# docker process. +# """ +# __client__ = docker.from_env() + +# def __init__(self, container_def): +# self._container_def = container_def + +# @property +# def container_def(self): +# return self._container_def + +# @staticmethod +# def remove_container(name): +# try: +# container = Containers.__client__.containers.get(name) +# container.kill() +# except docker.errors.NotFound: +# _log.debug(f"container {name} not found so couldn't remove.") + +# @staticmethod +# def container_list( +# ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): +# """ +# Provides a wrapper around the listing of docker containers. This function was +# brought about when running from within a docker container. + +# Currently the docker container that is run using docker-compose up from the +# gridappsd-dev-environment is specified as gridappsd_dev, however this can change +# and could potentially be extended to multiple named containers. + +# @param: ignore_list: +# optional container names to not stop :: A string or list of strings +# """ + +# if ignore_list is None: +# ignore_list = [] +# elif isinstance(ignore_list, str): +# ignore_list = [ignore_list] +# containers = [] +# for cont in Containers.__client__.containers.list(): +# if cont.name not in ignore_list: +# containers.append(cont) +# return containers + +# @staticmethod +# def reset_all_containers( +# ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): +# """ +# Provides a wrapper around the resetting of all docker containers. This function was +# brought about when running from within a docker container. + +# Currently the docker container that is run using docker-compose up from the +# gridappsd-dev-environment is specified as gridappsd_dev, however this can change +# and could potentially be extended to multiple named containers. + +# @param: ignore_list: +# optional container names to not stop :: A string or list of strings +# """ +# if ignore_list is None: +# ignore_list = [] +# elif isinstance(ignore_list, str): +# ignore_list = [ignore_list] + +# for cont in Containers.__client__.containers.list(): +# if cont.name not in ignore_list: +# cont.kill() + +# @staticmethod +# def check_required_running(self, config): +# my_config = deepcopy(config) + +# for c in Containers.__client__.containers.list(): +# my_config.pop(c.name, None) + +# assert not my_config, f"The required containers were not satisfied missing {list(my_config.keys())}" + +# @staticmethod +# def create_volume_container(name, +# volume_name, +# mount_in_container_at, +# restart_if_exists: bool = False, +# mode="rw"): + +# _log.info( +# f"Creating container {name} and mounting volume {volume_name} " +# f"at {mount_in_container_at} in container {name}") +# client = docker.from_env() + +# should_create = False +# try: +# cont = Containers.__client__.containers.get(name) +# except docker.errors.NotFound: +# # start container +# should_create = True +# else: +# if restart_if_exists: +# should_create = True +# cont.stop() +# else: +# return cont + +# if should_create: +# kwargs = {} +# kwargs['image'] = "alpine" +# # Only name the containers if remove is on +# kwargs['remove'] = True +# kwargs['name'] = name +# kwargs['detach'] = True +# kwargs['volumes'] = { +# volume_name: { +# "bind": mount_in_container_at, +# "mode": mode +# } +# } +# # keep the container running +# kwargs['command'] = "tail -f /dev/null" +# cont = client.containers.run(**kwargs) +# _log.info(f"New container for volume created {cont.name}") +# # print(f"New container is: {container.name}") +# return cont + +# @staticmethod +# def create_get_network(name: str) -> docker.models.networks.Network: +# try: +# network = Containers.__client__.networks.get(name) +# except docker.errors.NotFound: +# network = Containers.__client__.networks.create( +# name, driver="bridge") + +# return network + +# @staticmethod +# def copy_to(src: str, dst: str): +# """ +# Copy a local directory onto a destination + +# .. note:: + +# Make sure the dst (destination) is in the form container:directory + +# @param: src: The local directory to copy from the host +# @param: dst: The container:destination to copy the src to. +# """ +# _log.debug(f"copying folder from: {src} to {dst}") +# src = str(src) +# # python3.6 can't combine PosixPath and str +# assert os.path.exists(src), f"{src} does not exist!" +# client = docker.from_env() +# name, dst = dst.split(':') +# container = client.containers.get(name) +# assert container is not None +# cwd = os.getcwd() +# os.chdir(os.path.dirname(src)) +# srcname = os.path.basename(src) +# tarfilename = src + '.tar' +# tar = tarfile.open(tarfilename, mode='w') +# try: +# tar.add(srcname) +# finally: +# tar.close() + +# data = open(tarfilename, 'rb').read() +# resp = container.put_archive(os.path.dirname(dst), data) +# os.chdir(cwd) +# _log.debug(f"Response from put_archive {resp}") + +# def start(self): +# _log.info("Starting containers") +# # Containers.create_volume_container(CONFIGURATION_CONTAINER_NAME, "testconfig", "/testconfig") +# #pprint(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) +# client = docker.from_env() +# # print(f"Docker client version: {client.version()}") +# for service, value in self._container_def.items(): +# _log.info(f"Pulling {service} image") +# _log.info( +# f"Pulling {service} : {self._container_def[service]['image']}" +# ) +# client.images.pull(self._container_def[service]['image']) +# try: +# container = client.containers.get(service) +# self._container_def[service]['containerid'] = container.id +# except docker.errors.NotFound: +# _log.debug(f"Couldn't find {service} so continuing on.") + +# # Provide a way to dynamically create things that the container will need +# # on the host system. This is important if we want to create a volume before +# # starting the container up. +# if value.get('onsetupfn'): +# value.get('onsetupfn')() + +# if self._container_def[service].get("volumes_required"): +# for vr in self._container_def[service].get( +# "volumes_required"): +# _log.info( +# f"Creating volume for {service}: name={vr['name']}, " +# f"volume_name={vr['name']}, container_path={vr['container_path']}" +# ) +# Containers.create_volume_container( +# name=vr["name"], +# volume_name=vr["name"], +# mount_in_container_at=vr["container_path"], +# restart_if_exists=True) +# time.sleep(20) +# if vr.get("local_path"): +# _log.debug( +# f"contents of local path({vr.get('local_path')}):\n\t{os.listdir(vr.get('local_path'))}" +# ) +# _log.info( +# f"Copy to mounted volume for {service}: " +# f"local_path={vr['local_path']}, container_path={vr['container_path']}" +# ) +# Containers.copy_to( +# vr["local_path"], +# f'{vr["name"]}:{vr["container_path"]}') +# network = Containers.create_get_network(NETWORK) +# for service, value in self._container_def.items(): +# if self._container_def[service]['start']: +# _log.debug( +# f"Starting {service} : {self._container_def[service]['image']}" +# ) +# kwargs = {} +# kwargs['image'] = self._container_def[service]['image'] +# # Only name the containers if remove is on +# kwargs['remove'] = True +# kwargs['name'] = service +# kwargs['detach'] = True +# kwargs['entrypoint'] = value.get('entrypoint') +# if self._container_def[service].get('entrypoint'): +# kwargs['entrypoint'] = value['entrypoint'] +# if self._container_def[service].get('environment'): +# kwargs['environment'] = value['environment'] +# if self._container_def[service].get('ports'): +# kwargs['ports'] = value['ports'] +# if self._container_def[service].get('volumes'): +# kwargs['volumes'] = value['volumes'] +# if self._container_def[service].get('entrypoint'): +# kwargs['entrypoint'] = value['entrypoint'] +# # if self._container_def[service].get('links'): +# # kwargs['links'] = value['links'] +# if self._container_def[service].get('volumes_from'): +# kwargs['volumes_from'] = value['volumes_from'] +# #for k, v in kwargs.items(): +# # print(f"k->{k}, v->{v}") +# _log.debug("Starting container with the following args:") +# _log.debug(f"{pformat(kwargs)}") +# launched_container = None +# try: +# container = client.containers.get(service) +# _log.debug("Found existing container") +# except docker.errors.NotFound: +# container = client.containers.run(**kwargs) +# _log.debug("Started new container") +# network.connect(container.id) +# self._container_def[service]['containerid'] = container.id +# _log.debug( +# f"Current containers are: {[x.name for x in client.containers.list()]}" +# ) + +# def wait_for_log_pattern(self, container, pattern, timeout=60): +# assert self._container_def.get( +# container), f"Container {container} is not in definition." +# client = docker.from_env() +# container = client.containers.get( +# self._container_def.get(container)['containerid']) +# until = datetime.now() + timedelta(seconds=timeout) +# for p in container.logs(stream=True, until=until): +# _log.info(f"HANDLER: {p.decode('utf-8')}") +# if pattern in p.decode('utf-8'): +# print(f"Found pattern {pattern}") +# return +# raise TimeoutError( +# f"Pattern {pattern} was not found in logs of container {container} within {timeout}s" +# ) + +# def wait_for_http_ok(self, url, timeout=30): +# import requests +# results = None +# test_count = 0 + +# while results is None or not results.ok: +# test_count += 1 +# if test_count > timeout: +# raise TimeoutError( +# f"Could not reach {url} within allotted timeout {timeout}s" +# ) +# results = requests.get(url) +# time.sleep(1) + +# print(f"Found url {url} within timeout {timeout}") + +# def stop(self): +# client = docker.from_env() +# for service, value in self._container_def.items(): +# if value.get('containerid'): +# try: +# cnt = client.containers.get(value.get('containerid')) +# cnt.kill() +# time.sleep(2) + +# if "volumes_required" in value: +# # Loop over the volumes that are required for each image and +# # remove the volume. +# for volume_spec in value["volumes_required"]: +# Containers.remove_container( +# volume_spec['name']) +# time.sleep(2) + +# # client.containers.get(value.get('containerid')).kill() # value.get('name')).kill() +# except docker.errors.NotFound as ex: +# _log.error( +# f"Volume {value.get('containerid')} was not found." +# ) +# _log.exception(ex) + +# threads = [] + +# def stream_container_log_to_file(container_name: str, logfile: str): +# client = docker.from_env() +# container = client.containers.list(filters=dict( +# name=container_name))[0] + +# print(container) + +# def log_output(): +# nonlocal container, logfile +# print(f"Starting to write to file {logfile}.") +# with open(logfile, 'wb') as fp: +# print(f"openfile") +# for p in container.logs(stream=True, stderr=PIPE, stdout=PIPE): +# fp.write(p) +# fp.flush() + +# threading.Thread(target=log_output, daemon=True).start() + +# @contextlib.contextmanager +# def run_containers(config, stop_after=True) -> Containers: +# containers = Containers(config) + +# containers.start() +# try: +# yield containers +# finally: +# if stop_after: +# containers.stop() + +# @contextlib.contextmanager +# def run_dependency_containers(stop_after=False) -> Containers: + +# containers = Containers(DEFAULT_DOCKER_DEPENDENCY_CONFIG) + +# containers.start() +# try: +# yield containers +# finally: +# if stop_after: +# containers.stop() + +# @contextlib.contextmanager +# def run_gridappsd_container(stop_after=True, +# rebuild_if_present=False) -> Container: +# """ A contextmanager that uses """ + +# parent_container = get_docker_in_docker() + +# client = docker.from_env() +# # if there is a parent_container then we need to make sure that it is connected +# # to the same network as our systems. If not then we need to modify the network +# # to include the parent container +# if parent_container: +# env = DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd'].get( +# 'environment') +# if env is None: +# env = {} +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.name] = 'gridappsd' +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.name] = 'test_app_user' +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.name] = '4Test' +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd']['environment'] = env + +# _log.debug( +# f"Running inside a container environment: {parent_container.name}" +# ) +# network = client.networks.get(NETWORK) +# has_it = False +# for x in network.containers: +# if x.name == parent_container.name: +# has_it = True +# _log.debug( +# f"parent_container {parent_container.name} is connected to the network." +# ) +# break +# if not has_it: +# _log.debug( +# f"Connecting new container to the network: {parent_container.name}" +# ) +# network.connect(parent_container) +# else: +# _log.debug("Not running in a container") + +# containers = Containers(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) + +# gridappsd_container = None +# try: +# gridappsd_container = client.containers.get("gridappsd") +# _log.info(f"{gridappsd_container.name} container found") +# if rebuild_if_present: +# gridappsd_container.kill() +# except docker.errors.NotFound: +# _log.debug("gridappsd container not found!") + +# try: +# if gridappsd_container is None: +# containers.start() + +# # the gridappsd container itself will take a bit to start up. +# time.sleep(30) + +# tries = 30 +# while True: +# tries -= 1 +# if tries <= 0: +# raise RuntimeError( +# "Couldn't connect to gridappsd server in a timely manner!" +# ) +# try: +# g = GridAPPSD() +# if g.connected: +# _log.info("Connected to gridappsd!") +# g.disconnect() +# break + +# except stomp.exception.ConnectFailedException or stomp.exception.NotConnectedException: +# _log.error("Retesting connection") + +# yield gridappsd_container +# finally: +# if stop_after: +# containers.stop() + +# def get_docker_in_docker() -> Container: +# """ +# Grab the parent container named the same as the current machine's hostname. We are assuming that this +# is going to be a container. +# """ +# # There needs to be a test to make sure that the current container (assuming run in dev environment) +# # is able to be run from the docker dev environment. That environment is going to be assumed to be +# # the same name as the host name of the container. See the docker-compose starting the dev environment +# hostname = str(open("/etc/hostname", "rt").read().strip()) +# client = docker.from_env() +# try: +# parent_container = client.containers.get(hostname) +# _log.info(f"Inside parent container: {hostname}") +# # Setup to use gridappsd as the connection address. This value is used in the +# # utils script to establish connection with the gridappsd server +# os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" +# except docker.errors.NotFound: +# _log.debug( +# f"Docker container is not named this hostname {hostname}") +# parent_container = None +# return parent_container diff --git a/gridappsd/goss.py b/gridappsd-python-lib/gridappsd/goss.py similarity index 99% rename from gridappsd/goss.py rename to gridappsd-python-lib/gridappsd/goss.py index 6757e67..8dff23f 100644 --- a/gridappsd/goss.py +++ b/gridappsd-python-lib/gridappsd/goss.py @@ -356,7 +356,7 @@ def __init__(self): self._thread.start() def run_callbacks(self): - _log.info("Starting thread queue") + _log.debug("Starting thread queue") while True: cb, hdrs, msg = self._queue_callerback.get() try: diff --git a/gridappsd/gridappsd.py b/gridappsd-python-lib/gridappsd/gridappsd.py similarity index 100% rename from gridappsd/gridappsd.py rename to gridappsd-python-lib/gridappsd/gridappsd.py diff --git a/gridappsd/houses.py b/gridappsd-python-lib/gridappsd/houses.py similarity index 100% rename from gridappsd/houses.py rename to gridappsd-python-lib/gridappsd/houses.py diff --git a/gridappsd/loghandler.py b/gridappsd-python-lib/gridappsd/loghandler.py similarity index 100% rename from gridappsd/loghandler.py rename to gridappsd-python-lib/gridappsd/loghandler.py diff --git a/gridappsd/register_app.py b/gridappsd-python-lib/gridappsd/register_app.py similarity index 100% rename from gridappsd/register_app.py rename to gridappsd-python-lib/gridappsd/register_app.py diff --git a/gridappsd/simulation.py b/gridappsd-python-lib/gridappsd/simulation.py similarity index 100% rename from gridappsd/simulation.py rename to gridappsd-python-lib/gridappsd/simulation.py diff --git a/gridappsd/timeseries.py b/gridappsd-python-lib/gridappsd/timeseries.py similarity index 100% rename from gridappsd/timeseries.py rename to gridappsd-python-lib/gridappsd/timeseries.py diff --git a/gridappsd/topics.py b/gridappsd-python-lib/gridappsd/topics.py similarity index 71% rename from gridappsd/topics.py rename to gridappsd-python-lib/gridappsd/topics.py index 22f3722..87bf567 100644 --- a/gridappsd/topics.py +++ b/gridappsd-python-lib/gridappsd/topics.py @@ -44,6 +44,10 @@ FNCS_BASE_OUTPUT_TOPIC = '/topic/goss.gridappsd.simulation.output' BASE_SIMULATION_TOPIC = '/topic/goss.gridappsd.simulation' BASE_SIMULATION_LOG_TOPIC = "/topic/goss.gridappsd.simulation.log" +BASE_FIELD_TOPIC = '/topic/goss.gridappsd.field' + +BASE_FIELD_QUEUE = 'goss.gridappsd.field' +REGISTER_AGENT_QUEUE = 'goss.gridappsd.field.register.agent' BLAZEGRAPH = "/queue/goss.gridappsd.process.request.data.powergridmodel" # https://gridappsd.readthedocs.io/en/latest/using_gridappsd/index.html#querying-logs @@ -69,7 +73,6 @@ REQUEST_APP_START = ".".join((PROCESS_PREFIX, "request.app.start")) BASE_APPLICATION_HEARTBEAT = ".".join((BASE_TOPIC_PREFIX, "heartbeat")) - def platform_log_topic(): """ Utility method for getting the platform.log base topic """ @@ -171,3 +174,86 @@ def simulation_log_topic(simulation_id): """https://gridappsd.readthedocs.io/en/latest/using_gridappsd/index.html#subscribing-to-logs """ return "{}.{}".format(BASE_SIMULATION_LOG_TOPIC, simulation_id) + +def field_message_bus_topic(message_bus_id:str, app_id: str=None, agent_id: str=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param app_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + + return f"{BASE_FIELD_TOPIC}.{message_bus_id}.{app_id}.{agent_id}" + + +def field_message_bus_app_topic(message_bus_id, app_id=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param app_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, app_id) + +def field_message_bus_agent_topic(message_bus_id, agent_id=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, agent_id) + +def field_agent_request_queue(message_bus_id, agent_id): + """ Utility method for getting the request topic for a specific distributed agent + + :param message_bus_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, agent_id) + +def context_request_queue(message_bus_id): + """ Utility method for getting the request topic for context manager + + :param message_bus_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + + return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, message_bus_id+'.context_manager') + +def field_output_topic(message_bus_id=None, simulation_id=None): + """ Utility method for getting the field output topic. + If message_bus_id is None, it returns topic used by centralized device interfaces to publish measurements. + If message_bus_id is not None, it returns topic used by distributed devices interfaces to publish measurements which is then subscribed by distributed agents. + + :param message_bus_id: + :param simulation_id + :return: str: Topic to receive field measurements + """ + + if simulation_id is None: + return "{}.{}".format(BASE_FIELD_TOPIC, "output") + else: + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.output",simulation_id,message_bus_id) + +def field_input_topic(message_bus_id=None, simulation_id=None): + """ Utility method for getting the field input topic. + If message_bus_id is None, it returns topic used by centralized device interfaces to subscribe to control commands. + If message_bus_id is not None, it returns topic used by distributed devices interfaces to subscribe to control commands. + + :param message_bus_id: + :param simulation_id + :return: str: Topic to receive input control commands + """ + + if simulation_id is None: + return "{}.{}".format(BASE_FIELD_TOPIC, "input") + else: + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.input",simulation_id,message_bus_id) diff --git a/gridappsd/utils.py b/gridappsd-python-lib/gridappsd/utils.py similarity index 100% rename from gridappsd/utils.py rename to gridappsd-python-lib/gridappsd/utils.py diff --git a/gridappsd-python-lib/info/CHANGELOG.md b/gridappsd-python-lib/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-python-lib/info/VERSION b/gridappsd-python-lib/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/gridappsd-python-lib/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-python-lib/poetry.toml b/gridappsd-python-lib/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/gridappsd-python-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml new file mode 100644 index 0000000..40cafad --- /dev/null +++ b/gridappsd-python-lib/pyproject.toml @@ -0,0 +1,50 @@ +[tool.poetry] +name = "gridappsd-python" +version = "2023.5.2a0" +description = "A GridAPPS-D Python Adapter" +authors = [ + "C. Allwardt <3979063+craig8@users.noreply.github.com>", + "P. Sharma =1.2.0"] +build-backend = "poetry.core.masonry.api" diff --git a/gridappsd-python-lib/tests/conftest.py b/gridappsd-python-lib/tests/conftest.py new file mode 100644 index 0000000..5319cc8 --- /dev/null +++ b/gridappsd-python-lib/tests/conftest.py @@ -0,0 +1,147 @@ +import logging +import os +import shutil +import sys +import time +from pathlib import Path + +import git +import pytest +from python_on_whales import docker +from python_on_whales.docker_client import DockerClient + +from gridappsd import GOSS, GridAPPSD + +# from gridappsd.docker_handler import (Containers, run_dependency_containers, +# run_gridappsd_container) + +levels = dict(CRITICAL=50, + FATAL=50, + ERROR=40, + WARNING=30, + WARN=30, + INFO=20, + DEBUG=10, + NOTSET=0) + +# Get string representation of the log level passed +LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG") + +# Make sure the level passed is one of the valid levels. +if LOG_LEVEL not in levels.keys(): + raise AttributeError("Invalid LOG_LEVEL environmental variable set.") + +# Set the numeric version of log level to pass to the basicConfig function +LOG_LEVEL_INT = levels[LOG_LEVEL] + +logging.basicConfig(stream=sys.stdout, + level=LOG_LEVEL_INT, + format="%(asctime)s|%(levelname)s|%(name)s|%(message)s") +logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +logging.getLogger("docker.utils.config").setLevel(logging.INFO) +logging.getLogger("docker.auth").setLevel(logging.INFO) + +_log = logging.getLogger(__name__) + +STOP_CONTAINER_AFTER_TEST = os.environ.get( + 'GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) +os.environ['GRIDAPPSD_USER'] = 'system' +os.environ['GRIDAPPSD_PASSWORD'] = 'manager' +os.environ['GRIDAPPSD_TAG'] = ':develop' +os.environ["GRIDAPPSD_ADDRESS"] = "localhost" +os.environ["GRIDAPPSD_PORT"] = "61613" + +gridappsd_docker_url = "https://github.com/GRIDAPPSD/gridappsd-docker" +gridappsd_docker_clone_path = Path("/tmp/gridappsd-docker") + + +def clone_gridappsd_docker(): + + if gridappsd_docker_clone_path.exists(): + # TODO: Do something with the repo...checkout revert etc. + pass + #repo = git.Repo(str(gridappsd_docker_clone_path)) + else: + git.Repo.clone_from(url=gridappsd_docker_url, + to_path=str(gridappsd_docker_clone_path), + branch="main", + depth=1) + + +@pytest.fixture(scope="session") +def docker_compose_up() -> DockerClient: + clone_gridappsd_docker() + path = str(gridappsd_docker_clone_path / "docker-compose.yml") + assert os.path.exists(path) + dc = DockerClient(compose_files=[path]) + + config = dc.compose.config(return_json=False) + if "sample_app" in config.services: + assert config.services["sample_app"] + del config.services["sample_app"] + + dc.compose.up(detach=True, start=True, wait=True, recreate=True) + should_not_exit = True + + while should_not_exit: + log_stream = dc.logs(container="gridappsd", stream=True) + + for stream_type, stream_content in log_stream: + decoded_content = stream_content.decode('utf-8').strip() + if "gridappsd-topology-processor|None|STARTED" in decoded_content: + should_not_exit = False + break + _log.debug(decoded_content) + time.sleep(0.2) + + yield dc + + dc.compose.down() + # shutil.rmtree(gridappsd_docker_clone_path, ignore_errors=True) + + +# @pytest.fixture(scope="module") +# def docker_dependencies(): +# print("Docker dependencies") +# # Containers.reset_all_containers() + +# with run_dependency_containers( +# stop_after=STOP_CONTAINER_AFTER_TEST) as dep: +# yield dep +# print("Cleanup docker dependencies") + + +@pytest.fixture +def gridappsd_client(request, docker_compose_up: DockerClient): + + dc = docker_compose_up + + + gappsd = GridAPPSD() + assert gappsd.connected + + yield gappsd + + gappsd.disconnect() + + # with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + # gappsd = GridAPPSD() + # gappsd.connect() + # assert gappsd.connected + # models = gappsd.query_model_names() + # assert models is not None + # if request.cls is not None: + # request.cls.gridappsd_client = gappsd + # yield gappsd + + # gappsd.disconnect() + + +# @pytest.fixture +# def goss_client(docker_dependencies): +# with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): +# goss = GOSS() +# goss.connect() +# assert goss.connected + +# yield goss diff --git a/gridappsd-python-lib/tests/run_gridappsd.py b/gridappsd-python-lib/tests/run_gridappsd.py new file mode 100644 index 0000000..fd113e2 --- /dev/null +++ b/gridappsd-python-lib/tests/run_gridappsd.py @@ -0,0 +1,32 @@ +import logging +import os +import socket +import time + +import stomp + +from gridappsd import GridAPPSD + +logging.basicConfig(level=logging.DEBUG) + +_log = logging.getLogger(__name__) + +os.environ["GRIDAPPSD_USER"] = "system" +os.environ["GRIDAPPSD_PASSWORD"] = "manager" +os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" +os.environ["GRIDAPPSD_PORT"] = "61613" + +gapps = None + +while gapps is None: + try: + gapps = GridAPPSD() + except (ConnectionRefusedError, stomp.exception.ConnectFailedException, + socket.gaierror, OSError): + _log.debug("Not Connected") + time.sleep(5) + else: + _log.debug("Connected!") + +_log.debug("Complete!") +gapps.disconnect() diff --git a/tests/simulation_fixtures/13_node_2_min_base.json b/gridappsd-python-lib/tests/simulation_fixtures/13_node_2_min_base.json similarity index 100% rename from tests/simulation_fixtures/13_node_2_min_base.json rename to gridappsd-python-lib/tests/simulation_fixtures/13_node_2_min_base.json diff --git a/gridappsd-python-lib/tests/test_containers.py b/gridappsd-python-lib/tests/test_containers.py new file mode 100644 index 0000000..4680cb5 --- /dev/null +++ b/gridappsd-python-lib/tests/test_containers.py @@ -0,0 +1,101 @@ +# import logging +# import os +# from pathlib import Path +# import random +# import shutil +# import sys +# from unittest import TestCase + +# _log = logging.getLogger("test_containers") + +# try: +# from python_on_whales import docker +# HAS_DOCKER = True +# except ImportError: +# _log.warning("Docker api not loaded. pip install docker to install as package.") +# HAS_DOCKER = False + +# from gridappsd.docker_handler import Containers + +# class ContainersTestCase(TestCase): + +# @classmethod +# def setUpClass(cls) -> None: +# logging.basicConfig(level=logging.DEBUG,stream=sys.stdout) +# logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +# logging.getLogger("docker.utils.config").setLevel(logging.INFO) +# logging.getLogger("docker.auth").setLevel(logging.INFO) +# cls.log = logging.getLogger("test_containers") + +# cls.tmp_dir = Path("/tmp/tmpdir") +# os.makedirs(cls.tmp_dir, exist_ok=True) +# cls.tmp_file_name = cls.tmp_dir.joinpath("woot.txt") +# cls.tmp_file_content = """ +# here I come to save the day! +# """ +# with open(cls.tmp_file_name, "w") as stream: +# stream.write(cls.tmp_file_content) + +# def setUp(self) -> None: +# self.cname = f"test_container_{random.randint(1,1000)}" +# self.vname = f"test_volume_{random.randint(1,1000)}" +# self.in_container_path = "/foo/bar/bim" +# self.network_name = f"foo_{random.randint(1,1000)}" +# self.client = docker.from_env() + +# def test_can_create_volume(self): +# # container_path = "/foo/bar/bim" +# container = Containers.create_volume_container(name=self.cname, +# volume_name=self.vname, +# mount_in_container_at=self.in_container_path) +# assert self.cname == container.name +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") +# self.log.debug(f"Exit code: {exit_code}, result: {result}") +# assert exit_code == 0 + +# def test_can_copy_dir(self): +# Containers.create_volume_container(name=self.cname, +# volume_name=self.vname, +# mount_in_container_at=self.in_container_path) + +# Containers.copy_to(self.tmp_dir, f"{self.cname}:/foo/bar/bim/tmpdir") + +# container = self.client.containers.get(self.cname) + +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") + +# assert exit_code == 0 +# assert b"tmpdir" in result +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}/tmpdir") +# assert exit_code == 0 +# assert b"woot.txt" in result + +# def test_network_creation(self): +# network = Containers.create_get_network(self.network_name) +# assert network is not None +# assert self.network_name == network.name + +# def tearDown(self) -> None: +# try: +# container = self.client.containers.get(self.cname) +# container.stop() +# except docker.errors.NotFound: +# pass + +# try: +# volume = self.client.volumes.get(self.vname) +# volume.remove() +# except docker.errors.NotFound: +# pass + +# try: +# network = self.client.networks.get(self.network_name) +# network.remove() +# except docker.errors.NotFound: +# pass +# self.log.debug("tearDown") + +# @classmethod +# def tearDownClass(cls) -> None: +# shutil.rmtree(cls.tmp_dir, ignore_errors=True) +# cls.log.debug("tearDownClass") diff --git a/gridappsd-python-lib/tests/test_docker_handler.py b/gridappsd-python-lib/tests/test_docker_handler.py new file mode 100644 index 0000000..4f8826b --- /dev/null +++ b/gridappsd-python-lib/tests/test_docker_handler.py @@ -0,0 +1,197 @@ +# import inspect +# import logging +# import os +# import sys +# import time +# from pathlib import Path + +# from python_on_whales import docker + +# from gridappsd import GridAPPSD +# from gridappsd.docker_handler import (DEFAULT_DOCKER_DEPENDENCY_CONFIG, +# MYSQL_SCHEMA_INIT_DIR, Containers, +# mysql_setup, run_dependency_containers, +# run_gridappsd_container, +# stream_container_log_to_file) + +# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +# _log = logging.getLogger(inspect.getmodulename(__file__)) + + +# def test_log_container(docker_dependencies): +# mypath = "/tmp/alphabetagamma.txt" +# stream_container_log_to_file("influxdb", mypath) +# time.sleep(5) +# print("After call to stream") +# assert os.path.exists(mypath) +# with open(mypath, 'rb') as rf: +# assert len(rf.readlines()) > 0 + + +# def test_can_reset_all_containers(): +# Containers.reset_all_containers() +# assert not Containers.container_list() + +# config = { +# "redis": { +# "start": True, +# "image": "redis:3.2.11-alpine", +# "pull": True, +# "ports": { +# "6379/tcp": 6379 +# }, +# "environment": [], +# "links": "", +# "volumes": "", +# "entrypoint": "redis-server --appendonly yes", +# } +# } +# cont = Containers(config) +# cont.start() +# assert len(Containers.container_list()) == 1 +# time.sleep(5) +# Containers.reset_all_containers() +# assert not Containers.container_list() + + +# def test_can_dependencies_continue_after_context_manager(): +# my_config = DEFAULT_DOCKER_DEPENDENCY_CONFIG.copy() +# Containers.reset_all_containers() + +# time.sleep(3) +# my_dep_containers = None +# with run_dependency_containers() as containers: +# my_dep_containers = containers +# time.sleep(10) + +# real_containers = Containers.container_list() +# for k in my_dep_containers.container_def.keys(): +# found = False +# for c in real_containers: +# if c.name == k: +# found = True +# break +# assert found, f"Couldn't find {k} container in list" + +# Containers.reset_all_containers() + + +# def test_create_volume_container(): +# Containers.create_volume_container("test_volume", +# "test_volume", +# "/startup", +# restart_if_exists=True) +# path = str(Path("gridappsd/conf").absolute()) +# Containers.copy_to(path, "test_volume:/startup/conf") +# client = docker.from_env() +# result = client.containers.get("test_volume").exec_run("ls -l /startup") +# assert True + + +# def test_can_upload_files_to_container(): +# Containers.reset_all_containers() + +# client = docker.from_env() +# client.images.pull("alpine") +# test_container = client.containers.run(image="alpine", +# command="tail -f /dev/null", +# detach=True, +# name="test_upload_container", +# remove=True) +# # may take a few for image to be up +# time.sleep(20) +# conf_path = str(Path("gridappsd/conf").absolute()) +# Containers.copy_to(conf_path, f"{test_container.name}:/conf") +# results = test_container.exec_run("ls -l /conf") +# for f in os.listdir(conf_path): +# assert f in results.output.decode("utf-8"), f"{f} was not in /conf" + + +# def test_multiple_runs_in_a_row_with_dependency_context_manager(): + +# Containers.reset_all_containers() + +# with run_dependency_containers(): +# pass + +# containers = [ +# x for x in Containers.container_list() if "config" not in x.name +# ] +# assert len(containers) == 5 + +# with run_gridappsd_container(): +# timeout = 0 +# gapps = None + +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + +# with run_gridappsd_container(): +# timeout = 0 +# gapps = None +# time.sleep(10) +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + + +# def test_can_start_gridappsd_within_dependency_context_manager_all_cleanup(): + +# Containers.reset_all_containers() + +# with run_dependency_containers(True) as cont: +# # True in this method will remove the containsers +# with run_gridappsd_container(True) as dep_con: +# # Default cleanup is true within run_gridappsd_container method +# timeout = 0 +# gapps = None +# time.sleep(10) +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + +# # Filter out the two config containers that we start up for volume data. +# containers = [ +# x.name for x in Containers.container_list() if "config" not in x.name +# ] +# assert not len(containers) + + +# def test_can_start_gridapps(): +# Containers.reset_all_containers() +# with run_dependency_containers() as cont: +# with run_gridappsd_container() as cont2: +# g = GridAPPSD() +# assert g.connected + + +# def test_mysql_setup(): +# mysql_setup() +# assert Path(MYSQL_SCHEMA_INIT_DIR).exists() +# assert Path(MYSQL_SCHEMA_INIT_DIR).joinpath( +# "gridappsd_mysql_dump.sql").is_file() diff --git a/gridappsd-python-lib/tests/test_goss.py b/gridappsd-python-lib/tests/test_goss.py new file mode 100644 index 0000000..7d5c4f4 --- /dev/null +++ b/gridappsd-python-lib/tests/test_goss.py @@ -0,0 +1,212 @@ +# import json +# import logging +# import os +# import threading +# from queue import Queue + +# import mock +# import pytest +# from time import sleep + +# from gridappsd import GOSS +# from gridappsd.docker_handler import get_docker_in_docker +# from gridappsd.goss import GRIDAPPSD_ENV_ENUM + +# _log = logging.getLogger(__name__) + + +# def test_auth_raises_error_no_username_password(docker_dependencies): +# container = get_docker_in_docker() +# mockdict = { +# GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.value: '', +# GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.value: '' +# } +# if container: +# mockdict[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.value] = "gridappsd" + +# with mock.patch.dict(os.environ, mockdict): +# with pytest.raises(ValueError) as ex: +# goss = GOSS() + +# with pytest.raises(ValueError) as ex: +# goss = GOSS(username="foo") + +# with pytest.raises(ValueError) as ex: +# goss = GOSS(password="bar") + + +# def test_get_response(caplog, goss_client): +# caplog.set_level(logging.DEBUG) + +# def addem_callback(header, message): +# print("Addem callback") +# print("Threadid: {}".format(threading.current_thread().ident)) + +# if isinstance(message, str): +# item = json.loads(message) +# else: +# item = message +# total = 0 +# for x in item: +# total += x + +# reply_to = header['reply-to'] +# response = dict(result=total) +# print("Sending back topic: {topic} {response}".format(topic=reply_to, +# response=response)) +# goss_client.send(reply_to, json.dumps(response)) + +# gen_sub = [] + +# def generic_subscription(header, message): +# gen_sub.append((header, message)) + +# # Simulate an rpc call. +# goss_client.subscribe("/addem", addem_callback) + +# goss_client.subscribe("foo", generic_subscription) + +# # id_before = id(goss_client._conn) +# result = goss_client.get_response('/addem', [5, 6]) +# assert result['result'] == 11 +# # assert id_before == id(goss_client._conn) + +# goss_client.send("foo", str(result['result'])) + +# count = 0 +# while True: +# sleep(0.1) +# count += 1 +# if len(gen_sub) > 0 or count > 10: +# break + +# assert gen_sub +# assert len(gen_sub) == 1 +# assert len(gen_sub[0]) == 2 +# assert result['result'] == 11 + + +# def test_send_receive(goss_client): +# message_queue = Queue() + +# class MyListener(object): +# def on_message(self, headers, message): +# message_queue.put((headers, message)) + +# listener = MyListener() +# goss_client.subscribe('doah', listener) +# goss_client.send('doah', "I am a foo") +# sleep(0.5) +# assert message_queue.qsize() == 1 +# header, message = message_queue.get() +# assert message == "I am a foo" + + +# def test_callback_function(goss_client): +# message_queue1 = Queue() + +# def callback1(headers, message): +# message_queue1.put((headers, message)) + +# goss_client.subscribe('foo', callback1) +# goss_client.send('foo', "I am a foo") +# sleep(0.5) +# assert message_queue1.qsize() == 1 +# header, message = message_queue1.get() +# assert message == "I am a foo" + + +# def test_multi_subscriptions(goss_client): +# message_queue1 = Queue() +# message_queue2 = Queue() + +# def callback1(headers, message): +# print(f"mq1 {headers} {message}") +# message_queue1.put((headers, message)) + +# def callback2(headers, message): +# print(f"mq2 {headers} {message}") +# message_queue2.put((headers, message)) + +# goss_client.subscribe('bim', callback1) +# goss_client.subscribe('bar', callback2) +# sleep(0.5) +# goss_client.send('bim', "I am a foo") +# goss_client.send('bar', "I am a bar") +# sleep(0.5) +# assert message_queue1.qsize() == 1 +# assert message_queue2.qsize() == 1 +# header, message = message_queue1.get() +# assert message == "I am a foo" +# header, message = message_queue2.get() +# assert message == "I am a bar" + + +# def test_multi_subscriptions_same_topic(goss_client): +# # pytest.xfail("Multiple topics can't be subscribed to the same topic at present.") + +# message_queue1 = Queue() +# message_queue2 = Queue() + +# def callback1(headers, message): +# print(f"handling callback1 {message} ") +# message_queue1.put((headers, message)) + +# def callback2(headers, message): +# print(f"handling callback2 {message} ") +# message_queue2.put((headers, message)) + +# indx1 = goss_client.subscribe('bim', callback1) +# goss_client.subscribe('bim', callback2) +# sleep(0.5) +# goss_client.send('bim', "I am a foo") +# goss_client.send('bim', "I am a bar") +# sleep(0.5) +# assert message_queue1.qsize() == 2 +# assert message_queue2.qsize() == 2 +# header, message = message_queue1.get() +# assert message == "I am a foo" +# header, message = message_queue1.get() +# assert message == "I am a bar" +# header, message = message_queue2.get() +# assert message == "I am a foo" +# header, message = message_queue2.get() +# assert message == "I am a bar" + + +# def test_response_class(goss_client): +# message_queue = Queue() + +# class SubListener: +# def on_message(self, header, message): +# message_queue.put((header, message)) + +# goss_client.subscribe("/topic/bar", SubListener()) +# sleep(0.5) +# goss_client.send("/topic/bar", {"abc": "def"}) + +# result = message_queue.get() + +# print(result) +# assert result +# assert 2 == len(result) + +# assert dict(abc="def") == result[1] + + +# def test_replace_subscription(caplog, goss_client): +# caplog.set_level(logging.DEBUG) +# original_queue = Queue() +# after_queue = Queue() + +# def original_callback(headers, message): +# original_queue.put((headers, message)) + +# def after_callback(headers, message): +# after_queue.put((headers, message)) + +# goss_client.subscribe("woot", original_callback) +# goss_client.send("woot", "This is a message") +# sleep(0.5) + +# assert original_queue.qsize() == 1 diff --git a/gridappsd-python-lib/tests/test_gridappsd.py b/gridappsd-python-lib/tests/test_gridappsd.py new file mode 100644 index 0000000..6990cf8 --- /dev/null +++ b/gridappsd-python-lib/tests/test_gridappsd.py @@ -0,0 +1,141 @@ +# import logging +# import os +# import xml.etree.ElementTree as ET +# from time import sleep + +# import mock + +from gridappsd import GridAPPSD + +#, topics as t, ProcessStatusEnum + + +# def test_get_gridappsd_client(gridappsd_client: GridAPPSD): +# assert isinstance(gridappsd_client, GridAPPSD) + + +# def test_get_model_info(gridappsd_client): +# """ The expecation is that we will have multiple models that we can retrieve from the +# database. Two of which should have the model name of ieee8500 and ieee123. The models +# should have the correct entry keys. +# """ + +# gappsd = gridappsd_client +# import time +# time.sleep(10) +# info = gappsd.query_model_info() + +# node_8500 = None +# node_123 = None +# for info_def in info['data']['models']: +# if info_def['modelName'] == 'ieee8500': +# node_8500 = info_def +# elif info_def['modelName'] == 'ieee123': +# node_123 = info_def + +# assert node_123, "Missing the 123 model" +# assert node_8500, "Missing 8500 node model." + +# keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", +# "regionName", "regionId"] +# correct_keys = set(keys) + +# assert len(correct_keys) == len(node_123) +# assert len(correct_keys) == len(node_8500) + +# for x in node_123: +# correct_keys.remove(x) + +# assert len(correct_keys) == 0 + +# correct_keys = set(keys) + +# for x in node_8500: +# correct_keys.remove(x) + +# assert len(correct_keys) == 0 + +# def test_listener_multi_topic(gridappsd_client): +# gappsd = gridappsd_client + +# class Listener: +# def __init__(self): +# self.call_count = 0 + +# def reset(self): +# self.call_count = 0 + +# def on_message(self, headers, message): +# print("Message was: {}".format(message)) +# self.call_count += 1 + +# listener = Listener() + +# input_topic = t.simulation_input_topic("5144") +# output_topic = t.simulation_output_topic("5144") + +# gappsd.subscribe(input_topic, listener) +# gappsd.subscribe(output_topic, listener) + +# gappsd.send(input_topic, "Any message") +# sleep(1) +# assert 1 == listener.call_count +# listener.reset() +# gappsd.send(output_topic, "No big deal") +# sleep(1) +# assert 1 == listener.call_count + +# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", +# "GRIDAPPSD_SIMULATION_ID": "1234"}) +# def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): + +# class Listener: +# def __init__(self): +# self.call_count = 0 + +# def reset(self): +# self.call_count = 0 + +# def on_message(self, headers, message): +# print("Message was: {}".format(message)) +# self.call_count += 1 + +# listener = Listener() +# gappsd = gridappsd_client +# assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' +# assert gappsd.get_simulation_id() == "1234" + +# log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) +# gappsd.subscribe(log_topic, listener) +# gappsd.send_simulation_status("RUNNING", +# "testing the sending and recieving of send_simulation_status().", +# logging.DEBUG) +# sleep(1) +# assert listener.call_count == 1 + +# new_log_topic = t.simulation_log_topic("54232") +# gappsd.set_simulation_id(54232) +# gappsd.subscribe(new_log_topic, listener) +# gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") +# sleep(1) +# assert listener.call_count == 2 + +# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) +# def test_gridappsd_status(gridappsd_client): +# gappsd = gridappsd_client +# assert "helics_goss_bridge.py" == gappsd.get_application_id() +# assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value +# assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value +# gappsd.set_application_status("RUNNING") + +# assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value +# assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value + +# gappsd.set_service_status("COMPLETE") +# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value +# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value + +# # Invalid +# gappsd.set_service_status("Foo") +# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value +# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value diff --git a/tests/test_houses.py b/gridappsd-python-lib/tests/test_houses.py similarity index 100% rename from tests/test_houses.py rename to gridappsd-python-lib/tests/test_houses.py diff --git a/gridappsd-python-lib/tests/test_logging.py b/gridappsd-python-lib/tests/test_logging.py new file mode 100644 index 0000000..a4da7ee --- /dev/null +++ b/gridappsd-python-lib/tests/test_logging.py @@ -0,0 +1,122 @@ +# from gridappsd.loghandler import Logger +# from gridappsd import topics as t, ProcessStatusEnum +# import os +# import mock +# from mock import Mock +# import pytest + + +# def init_gapps_mock(simulation_id=None, application_id=None, process_status=None, service_id=None): +# gapps = Mock() + +# gapps.get_simulation_id.return_value = simulation_id +# gapps.get_application_id.return_value = application_id +# gapps.get_application_status.return_value = process_status +# gapps.get_process_id.return_value = service_id + +# return gapps + + +# #@mock.patch('gridappsd.utils.get_application_id') +# def test_required_application_id_set(): +# """ os.environ['GRIDAPPSD_APPLICATION_ID'] must be set to run.""" +# log = Logger(init_gapps_mock()) + +# with pytest.raises(AttributeError): +# log.debug("foo") + + +# def test_no_simulation_id_topic_or_application_id(): +# """If no simulation then the topic should be the platform log topic""" +# expected_topic = t.platform_log_topic() + +# gapps_mock = init_gapps_mock(application_id="my_app_id", +# process_status=ProcessStatusEnum.STARTING.value) +# log = Logger(gapps_mock) + +# log.debug("A message") + +# topic, message = gapps_mock.send.call_args.args + +# assert expected_topic == topic +# assert message['processStatus'] == ProcessStatusEnum.STARTING.value +# assert message['logMessage'] == 'A message' + + +# def test_platform_log(): + +# application_id = "my_app" +# gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) +# log = Logger(gapps_mock) + +# log.debug("foo") +# gapps_mock.send.assert_called_once() + +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args + +# assert message['source'] == application_id +# assert message['logLevel'] == 'DEBUG' +# assert message['logMessage'] == 'foo' +# assert message['processStatus'] == ProcessStatusEnum.STOPPING.value +# gapps_mock.send.reset_mock() + +# log.info("bar") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'INFO' == message['logLevel'] +# assert 'bar' == message['logMessage'] +# gapps_mock.send.reset_mock() + +# log.error("bim") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'ERROR' == message['logLevel'] +# assert 'bim' == message['logMessage'] +# gapps_mock.send.reset_mock() + +# log.warning("baf") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'WARN' == message['logLevel'] +# assert 'baf' == message['logMessage'] + + +# def test_invalid_log_level(): +# application_id = "my_app" +# gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) +# log = Logger(gapps_mock) + +# with pytest.raises(AttributeError): +# log.log("junk error", "BART") + + +# def test_topic_and_status_set_correctly(): + +# sim_id = "543" +# application_id = "wicked_good_app_id" +# mock_gapps = init_gapps_mock(simulation_id=sim_id, application_id=application_id, +# process_status=ProcessStatusEnum.RUNNING.value) + +# expected_topic = t.simulation_log_topic(sim_id) + +# log = Logger(mock_gapps) + +# log.debug("A message") + +# # During the call to debug we expect the send function to be +# # called on the mock object. Grab the arguments and then +# # make sure that they are what we expect. +# topic, message = mock_gapps.send.call_args.args + +# assert message['source'] == application_id +# assert topic == expected_topic +# assert message['processStatus'] == "RUNNING" + + diff --git a/gridappsd-python-lib/tests/test_logging_integration.py b/gridappsd-python-lib/tests/test_logging_integration.py new file mode 100644 index 0000000..adb1e2c --- /dev/null +++ b/gridappsd-python-lib/tests/test_logging_integration.py @@ -0,0 +1,97 @@ +# import os +# import time + +# import mock +# import pytest + +# from gridappsd import GridAPPSD, topics as t +# from gridappsd.loghandler import Logger + + +# @pytest.fixture +# def logger_and_gridapspd(gridappsd_client) -> (Logger, GridAPPSD): + +# logger = Logger(gridappsd_client) + +# yield logger, gridappsd_client + +# logger = None + + +# @mock.patch.dict(os.environ, +# dict(GRIDAPPSD_APPLICATION_ID='sample_app', +# GRIDAPPSD_APPLICATION_STATUS='RUNNING')) +# def test_log_stored(logger_and_gridapspd): +# logger, gapps = logger_and_gridapspd + +# log_data_map = [ +# (logger.debug, "A debug message", "DEBUG"), +# (logger.info, "An info message", "INFO"), +# (logger.error, "An error message", "ERROR"), +# (logger.error, "Another error message", "ERROR"), +# (logger.info, "Another info message", "INFO"), +# (logger.debug, "A debug message", "DEBUG") +# ] + +# assert gapps.connected + +# # Make the calls to debug +# for d in log_data_map: +# d[0](d[1]) + +# payload = { +# "query": "select * from log order by timestamp" +# } +# time.sleep(5) +# response = gapps.get_response(t.LOGS, payload, timeout=60) +# assert response['data'], "There were not any records returned." + +# for x in response['data']: +# if x['source'] != 'sample_app': +# continue +# expected = log_data_map.pop(0) +# assert expected[1] == x['log_message'] +# assert expected[2] == x['log_level'] + + +# SIMULATION_ID='54321' + +# #TODO Ask about loging api for simulations. +# @mock.patch.dict(os.environ, +# dict(GRIDAPPSD_APPLICATION_ID='new_sample_app', +# GRIDAPPSD_APPLICATION_STATUS='RUNNING', +# GRIDAPPSD_SIMULATION_ID=SIMULATION_ID)) +# def test_simulation_log_stored(logger_and_gridapspd): +# logger, gapps = logger_and_gridapspd + +# assert gapps.get_simulation_id() == SIMULATION_ID + +# log_data_map = [ +# (logger.debug, "A debug message", "DEBUG"), +# (logger.info, "An info message", "INFO"), +# (logger.error, "An error message", "ERROR"), +# (logger.error, "Another error message", "ERROR"), +# (logger.info, "Another info message", "INFO"), +# (logger.debug, "A debug message", "DEBUG") +# ] + +# assert gapps.connected + +# # Make the calls to debug +# for d in log_data_map: +# d[0](d[1]) + +# time.sleep(5) +# payload = { +# "query": "select * from log" +# } + +# response = gapps.get_response(t.LOGS, payload, timeout=60) +# assert response['data'], "There were not any records returned." + +# for x in response['data']: +# if x['source'] != 'new_sample_app': +# continue +# expected = log_data_map.pop(0) +# assert expected[1] == x['log_message'] +# assert expected[2] == x['log_level'] diff --git a/gridappsd-python-lib/tests/test_simulation.py b/gridappsd-python-lib/tests/test_simulation.py new file mode 100644 index 0000000..5be2be8 --- /dev/null +++ b/gridappsd-python-lib/tests/test_simulation.py @@ -0,0 +1,45 @@ +# import json +# # from pprint import pprint +# import logging +# import os +# import sys +# import time +# import pytest + +# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + +# from gridappsd import GridAPPSD, topics as t +# from gridappsd.simulation import Simulation + +# # The directory containing this file +# HERE = os.path.dirname(__file__) + + +# def base_config(): +# data = {"power_system_config":{"SubGeographicalRegion_name":"_ABEB635F-729D-24BF-B8A4-E2EF268D8B9E","GeographicalRegion_name":"_73C512BD-7249-4F50-50DA-D93849B89C43","Line_name":"_49AD8E07-3BF9-A4E2-CB8F-C3722F837B62"},"simulation_config":{"power_flow_solver_method":"NR","duration":120,"simulation_name":"ieee13nodeckt","simulator":"GridLAB-D","start_time":1605418946,"run_realtime":False,"simulation_output":{},"model_creation_config":{"load_scaling_factor":1.0,"triplex":"y","encoding":"u","system_frequency":60,"voltage_multiplier":1.0,"power_unit_conversion":1.0,"unique_names":"y","schedule_name":"ieeezipload","z_fraction":0.0,"i_fraction":1.0,"p_fraction":0.0,"randomize_zipload_fractions":False,"use_houses":False},"simulation_broker_port":51044,"simulation_broker_location":"127.0.0.1"},"application_config":{"applications":[]},"service_configs":[],"test_config":{"randomNum":{"seed":{"value":185213303967438},"nextNextGaussian":0.0,"haveNextNextGaussian":False},"events":[],"testInput":True,"testOutput":True,"appId":"","testId":"1468836560","testType":"simulation_vs_expected","storeMatches":False},"simulation_request_type":"NEW"} +# # with open("{HERE}/simulation_fixtures/13_node_2_min_base.json".format(HERE=HERE)) as fp: +# # data = json.load(fp) +# return data + + +# def test_simulation_no_duplicate_measurement_timestamps(gridappsd_client: GridAPPSD): +# num_measurements = 0 +# timestamps = set() + +# def measurement(sim, timestamp, measurement): +# nonlocal num_measurements +# num_measurements += 1 +# assert timestamp not in timestamps +# timestamps.add(timestamp) + +# gapps = gridappsd_client +# sim = Simulation(gapps, base_config()) +# sim.add_onmeasurement_callback(measurement) +# sim.start_simulation() +# sim.run_loop() + +# # did we get a measurement? +# assert num_measurements > 0 + +# # if empty then we know the simulation did not work. +# assert timestamps diff --git a/tests/unittest_test_template.py b/gridappsd-python-lib/tests/unittest_test_template.py similarity index 100% rename from tests/unittest_test_template.py rename to gridappsd-python-lib/tests/unittest_test_template.py diff --git a/gridappsd/docker_handler.py b/gridappsd/docker_handler.py deleted file mode 100644 index 5b3776b..0000000 --- a/gridappsd/docker_handler.py +++ /dev/null @@ -1,650 +0,0 @@ -#!/usr/bin/env python3 - -import contextlib -import logging -import os -import random -import re -import shutil -import tarfile -import threading -import urllib.request -from copy import deepcopy -from datetime import datetime, timedelta -from pathlib import Path -from pprint import pformat -from subprocess import PIPE -from typing import Optional, Union - -import stomp -import time -from docker.models.containers import Container - -from gridappsd import GridAPPSD -from gridappsd.goss import GRIDAPPSD_ENV_ENUM - -logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) -logging.getLogger("docker.auth").setLevel(logging.INFO) -logging.getLogger("docker.utils").setLevel(logging.INFO) - -_log = logging.getLogger("gridappsd.docker_handler") - - -try: - import docker - HAS_DOCKER = True -except ImportError: - _log.warning("Docker api not loaded. pip install docker to install as package.") - HAS_DOCKER = False - -if HAS_DOCKER: - - # The following variable is used for creating a volume for the gridappsd container - # to utilize. It allows the ability to use multiple containers to run tests - # along side each other. - GRIDAPPSD_CONFIG_VOLUME_NAME = f"gridappsd_config_{random.randint(1,100)}" - - # This named container will be used to hold configuration/folders so that other containers - # that start can use them. To use add "volume_from": [CONFIGURATION_CONTAINER_NAME] and - # the mount point within the container will also be within the service container. - CONFIGURATION_CONTAINER_NAME = "testconfig" - - def expand_all(user_path): - return os.path.expandvars(os.path.expanduser(user_path)) - - __TMP_ROOT__ = "/tmp/assets" - if Path(__TMP_ROOT__).exists(): - shutil.rmtree(__TMP_ROOT__, ignore_errors=True) - os.makedirs(__TMP_ROOT__) - - # This path needs to be the path to the repo where configuration files are located. - GRIDAPPSD_CONF_DIR = Path(__file__).resolve().parent.parent.joinpath("gridappsd/conf") - - assert Path(GRIDAPPSD_CONF_DIR).joinpath("entrypoint.sh").exists() - assert Path(GRIDAPPSD_CONF_DIR).joinpath("run-gridappsd.sh").exists() - - GRIDAPPSD_DATA_REPO = str(Path(__TMP_ROOT__).joinpath("mysql").resolve()) - - os.makedirs(GRIDAPPSD_DATA_REPO, exist_ok=True) - if not Path(GRIDAPPSD_DATA_REPO).is_dir(): - raise AttributeError(f"Invalid GRIDAPPSD_DATA_REPO couldn't make or doesn't exist {GRIDAPPSD_DATA_REPO}") - - MYSQL_SCHEMA_INIT_DIR = f'{GRIDAPPSD_DATA_REPO}/docker-entrypoint-initdb.d' - - - def mysql_setup(): - """ - Downloads gridappsd_mysql_dump.sql into the MYSQL_SCHEMA_INIT_DIR init directory. - This will then be mounted into the mysql container. - """ - # Downlaod mysql file - _log.debug("Downloading mysql data file from Bootstrap repository") - mysql_file = f'{MYSQL_SCHEMA_INIT_DIR}/gridappsd_mysql_dump.sql' - if os.path.isdir(mysql_file): - raise RuntimeError(f"mysql datafile is directory, delete {mysql_file} using sudo rm -rf {mysql_file}") - if not os.path.isdir(MYSQL_SCHEMA_INIT_DIR): - os.makedirs(MYSQL_SCHEMA_INIT_DIR, 0o0775, exist_ok=True) - urllib.request.urlretrieve( - 'https://raw.githubusercontent.com/GRIDAPPSD/Bootstrap/master/gridappsd_mysql_dump.sql', - filename=mysql_file) - - # Modify the mysql file to allow connections from gridappsd container - with open(mysql_file, "r") as sources: - lines = sources.readlines() - with open(mysql_file, "w") as sources: - for line in lines: - sources.write(re.sub(r'localhost', '%', line)) - assert Path(mysql_file).exists() - - # Use the environmental variable if specified otherwise use the develop tag. - DEFAULT_GRIDAPPSD_TAG = os.environ.get('GRIDAPPSD_TAG_ENV', 'develop') - - # Network to connect all of the containers up to by default. - NETWORK = "test_my_network" - - __TPL_DEPENDENCY_CONFIG__ = { - "influxdb": { - "start": True, - "image": "gridappsd/influxdb:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8086/tcp": 8086}, - "environment": {"INFLUXDB_DB": "proven"} - }, - "redis": { - "start": True, - "image": "redis:3.2.11-alpine", - "pull": True, - "ports": {"6379/tcp": 6379}, - "environment": [], - "entrypoint": "redis-server --appendonly yes", - }, - "blazegraph": { - "start": True, - "image": "gridappsd/blazegraph:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8080/tcp": 8889}, - "environment": [] - }, - "mysql": { - "start": True, - "image": "mysql/mysql-server:5.7", - "pull": True, - "ports": {"3306/tcp": 3306}, - "environment": { - "MYSQL_RANDOM_ROOT_PASSWORD": "yes", - "MYSQL_PORT": "3306" - }, - # "volumes": { - # "/home/gridappsd/test-assets": {"bind": "/whatthehell", "mode": "rw"} - # #"test-assets": {"bind": "/whatthehell", "mode": "rw"} - # #data_dir + "/dumps/": {"bind": "/docker-entrypoint-initdb.d/", "mode": "ro"} - # }, - # Our own so we can create - "volumes_required": [ - dict(name="mysql_config", - local_path=MYSQL_SCHEMA_INIT_DIR, - container_path="/docker-entrypoint-initdb.d") - ], - "volumes_from": [ - "mysql_config" - ], - "onsetupfn": mysql_setup - }, - "proven": { - "start": True, - "image": "gridappsd/proven:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8080/tcp": 18080}, - "environment": { - "PROVEN_SERVICES_PORT": "18080", - "PROVEN_SWAGGER_HOST_PORT": "localhost:18080", - "PROVEN_USE_IDB": "true", - "PROVEN_IDB_URL": "http://influxdb:8086", - "PROVEN_IDB_DB": "proven", - "PROVEN_IDB_RP": "autogen", - "PROVEN_IDB_USERNAME": "root", - "PROVEN_IDB_PASSWORD": "root", - "PROVEN_T3DIR": "/proven"}, - "links": {"influxdb": "influxdb"} - } - } - - __TPL_GRIDAPPSD_CONFIG__ = { - "gridappsd": { - "start": True, - "image": "gridappsd/gridappsd:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"61613/tcp": 61613, "61614/tcp": 61614, "61616/tcp": 61616}, - "environment": { - "PATH": "/gridappsd/bin:/gridappsd/lib:/gridappsd/services/fncsgossbridge/service:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin", - "DEBUG": 1, - "START": 1 - }, - "links": {"mysql": "mysql", - "influxdb": "influxdb", - "blazegraph": "blazegraph", - "proven": "proven", - "redis": "redis"}, - "volumes_required": [ - dict(name=GRIDAPPSD_CONFIG_VOLUME_NAME, - local_path=GRIDAPPSD_CONF_DIR, - container_path="/startup/conf") - ], - "volumes_from": [ - GRIDAPPSD_CONFIG_VOLUME_NAME - ], - "entrypoint": "/startup/conf/entrypoint.sh", - "command": "/startup/conf/entrypoint.sh" - } - } - - def __update_template_data__(data, update_dict): - data_cpy = deepcopy(data) - for k, v in data_cpy.items(): - for u, p in update_dict.items(): - v['image'] = v['image'].replace(u, p) - - return data_cpy - - - __replace_dict__ = {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG} - DEFAULT_DOCKER_DEPENDENCY_CONFIG = __update_template_data__(__TPL_DEPENDENCY_CONFIG__, __replace_dict__) - DEFAULT_GRIDAPPSD_DOCKER_CONFIG = __update_template_data__(__TPL_GRIDAPPSD_CONFIG__, __replace_dict__) - - def update_gridappsd_tag(new_gridappsd_tag): - """ - Update the default tag used within the dependency and gridappsd containers to be - what is specified in the new gridappsd_tag variable - """ - global DEFAULT_GRIDAPPSD_TAG - - DEFAULT_GRIDAPPSD_TAG = new_gridappsd_tag - _log.info(f"Updated gridappsd docker tag {DEFAULT_GRIDAPPSD_TAG} ") - __replace_dict__.update({"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG}) - DEFAULT_DOCKER_DEPENDENCY_CONFIG.update(__update_template_data__(__TPL_DEPENDENCY_CONFIG__, __replace_dict__)) - DEFAULT_GRIDAPPSD_DOCKER_CONFIG.update(__update_template_data__(__TPL_GRIDAPPSD_CONFIG__, __replace_dict__)) - - - class Containers: - """ - This class allows the creation/management of containers created by the gridappsd - docker process. - """ - __client__ = docker.from_env() - - def __init__(self, container_def): - self._container_def = container_def - - @property - def container_def(self): - return self._container_def - - @staticmethod - def remove_container(name): - try: - container = Containers.__client__.containers.get(name) - container.kill() - except docker.errors.NotFound: - _log.debug(f"container {name} not found so couldn't remove.") - - @staticmethod - def container_list(ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): - """ - Provides a wrapper around the listing of docker containers. This function was - brought about when running from within a docker container. - - Currently the docker container that is run using docker-compose up from the - gridappsd-dev-environment is specified as gridappsd_dev, however this can change - and could potentially be extended to multiple named containers. - - @param: ignore_list: - optional container names to not stop :: A string or list of strings - """ - - if ignore_list is None: - ignore_list = [] - elif isinstance(ignore_list, str): - ignore_list = [ignore_list] - containers = [] - for cont in Containers.__client__.containers.list(): - if cont.name not in ignore_list: - containers.append(cont) - return containers - - @staticmethod - def reset_all_containers(ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): - """ - Provides a wrapper around the resetting of all docker containers. This function was - brought about when running from within a docker container. - - Currently the docker container that is run using docker-compose up from the - gridappsd-dev-environment is specified as gridappsd_dev, however this can change - and could potentially be extended to multiple named containers. - - @param: ignore_list: - optional container names to not stop :: A string or list of strings - """ - if ignore_list is None: - ignore_list = [] - elif isinstance(ignore_list, str): - ignore_list = [ignore_list] - - for cont in Containers.__client__.containers.list(): - if cont.name not in ignore_list: - cont.kill() - - @staticmethod - def check_required_running(self, config): - my_config = deepcopy(config) - - for c in Containers.__client__.containers.list(): - my_config.pop(c.name, None) - - assert not my_config, f"The required containers were not satisfied missing {list(my_config.keys())}" - - @staticmethod - def create_volume_container(name, volume_name, - mount_in_container_at, restart_if_exists: bool = False, - mode="rw"): - - _log.info(f"Creating container {name} and mounting volume {volume_name} " - f"at {mount_in_container_at} in container {name}") - client = docker.from_env() - - should_create = False - try: - cont = Containers.__client__.containers.get(name) - except docker.errors.NotFound: - # start container - should_create = True - else: - if restart_if_exists: - should_create = True - cont.stop() - else: - return cont - - if should_create: - kwargs = {} - kwargs['image'] = "alpine" - # Only name the containers if remove is on - kwargs['remove'] = True - kwargs['name'] = name - kwargs['detach'] = True - kwargs['volumes'] = { - volume_name: {"bind": mount_in_container_at, "mode": mode} - } - # keep the container running - kwargs['command'] = "tail -f /dev/null" - cont = client.containers.run(**kwargs) - _log.info(f"New container for volume created {cont.name}") - # print(f"New container is: {container.name}") - return cont - - @staticmethod - def create_get_network(name: str) -> docker.models.networks.Network: - try: - network = Containers.__client__.networks.get(name) - except docker.errors.NotFound: - network = Containers.__client__.networks.create(name, driver="bridge") - - return network - - @staticmethod - def copy_to(src: str, dst: str): - """ - Copy a local directory onto a destination - - .. note:: - - Make sure the dst (destination) is in the form container:directory - - @param: src: The local directory to copy from the host - @param: dst: The container:destination to copy the src to. - """ - _log.debug(f"copying folder from: {src} to {dst}") - src = str(src) - # python3.6 can't combine PosixPath and str - assert os.path.exists(src), f"{src} does not exist!" - client = docker.from_env() - name, dst = dst.split(':') - container = client.containers.get(name) - assert container is not None - cwd = os.getcwd() - os.chdir(os.path.dirname(src)) - srcname = os.path.basename(src) - tarfilename = src + '.tar' - tar = tarfile.open(tarfilename, mode='w') - try: - tar.add(srcname) - finally: - tar.close() - - data = open(tarfilename, 'rb').read() - resp = container.put_archive(os.path.dirname(dst), data) - os.chdir(cwd) - _log.debug(f"Response from put_archive {resp}") - - def start(self): - _log.info("Starting containers") - # Containers.create_volume_container(CONFIGURATION_CONTAINER_NAME, "testconfig", "/testconfig") - #pprint(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) - client = docker.from_env() - # print(f"Docker client version: {client.version()}") - for service, value in self._container_def.items(): - _log.info(f"Pulling {service} image") - _log.info(f"Pulling {service} : {self._container_def[service]['image']}") - client.images.pull(self._container_def[service]['image']) - try: - container = client.containers.get(service) - self._container_def[service]['containerid'] = container.id - except docker.errors.NotFound: - _log.debug(f"Couldn't find {service} so continuing on.") - - # Provide a way to dynamically create things that the container will need - # on the host system. This is important if we want to create a volume before - # starting the container up. - if value.get('onsetupfn'): - value.get('onsetupfn')() - - if self._container_def[service].get("volumes_required"): - for vr in self._container_def[service].get("volumes_required"): - _log.info(f"Creating volume for {service}: name={vr['name']}, " - f"volume_name={vr['name']}, container_path={vr['container_path']}") - Containers.create_volume_container( - name=vr["name"], - volume_name=vr["name"], - mount_in_container_at=vr["container_path"], - restart_if_exists=True - ) - time.sleep(20) - if vr.get("local_path"): - _log.debug(f"contents of local path({vr.get('local_path')}):\n\t{os.listdir(vr.get('local_path'))}") - _log.info(f"Copy to mounted volume for {service}: " - f"local_path={vr['local_path']}, container_path={vr['container_path']}") - Containers.copy_to(vr["local_path"], f'{vr["name"]}:{vr["container_path"]}') - network = Containers.create_get_network(NETWORK) - for service, value in self._container_def.items(): - if self._container_def[service]['start']: - _log.debug(f"Starting {service} : {self._container_def[service]['image']}") - kwargs = {} - kwargs['image'] = self._container_def[service]['image'] - # Only name the containers if remove is on - kwargs['remove'] = True - kwargs['name'] = service - kwargs['detach'] = True - kwargs['entrypoint'] = value.get('entrypoint') - if self._container_def[service].get('entrypoint'): - kwargs['entrypoint'] = value['entrypoint'] - if self._container_def[service].get('environment'): - kwargs['environment'] = value['environment'] - if self._container_def[service].get('ports'): - kwargs['ports'] = value['ports'] - if self._container_def[service].get('volumes'): - kwargs['volumes'] = value['volumes'] - if self._container_def[service].get('entrypoint'): - kwargs['entrypoint'] = value['entrypoint'] - # if self._container_def[service].get('links'): - # kwargs['links'] = value['links'] - if self._container_def[service].get('volumes_from'): - kwargs['volumes_from'] = value['volumes_from'] - #for k, v in kwargs.items(): - # print(f"k->{k}, v->{v}") - _log.debug("Starting container with the following args:") - _log.debug(f"{pformat(kwargs)}") - launched_container = None - try: - container = client.containers.get(service) - _log.debug("Found existing container") - except docker.errors.NotFound: - container = client.containers.run(**kwargs) - _log.debug("Started new container") - network.connect(container.id) - self._container_def[service]['containerid'] = container.id - _log.debug(f"Current containers are: {[x.name for x in client.containers.list()]}") - - def wait_for_log_pattern(self, container, pattern, timeout=60): - assert self._container_def.get(container), f"Container {container} is not in definition." - client = docker.from_env() - container = client.containers.get(self._container_def.get(container)['containerid']) - until = datetime.now() + timedelta(seconds=timeout) - for p in container.logs(stream=True, until=until): - _log.info(f"HANDLER: {p.decode('utf-8')}") - if pattern in p.decode('utf-8'): - print(f"Found pattern {pattern}") - return - raise TimeoutError(f"Pattern {pattern} was not found in logs of container {container} within {timeout}s") - - def wait_for_http_ok(self, url, timeout=30): - import requests - results = None - test_count = 0 - - while results is None or not results.ok: - test_count += 1 - if test_count > timeout: - raise TimeoutError(f"Could not reach {url} within allotted timeout {timeout}s") - results = requests.get(url) - time.sleep(1) - - print(f"Found url {url} within timeout {timeout}") - - def stop(self): - client = docker.from_env() - for service, value in self._container_def.items(): - if value.get('containerid'): - try: - cnt = client.containers.get(value.get('containerid')) - cnt.kill() - time.sleep(2) - - if "volumes_required" in value: - # Loop over the volumes that are required for each image and - # remove the volume. - for volume_spec in value["volumes_required"]: - Containers.remove_container(volume_spec['name']) - time.sleep(2) - - # client.containers.get(value.get('containerid')).kill() # value.get('name')).kill() - except docker.errors.NotFound as ex: - _log.error(f"Volume {value.get('containerid')} was not found.") - _log.exception(ex) - - - threads = [] - - def stream_container_log_to_file(container_name: str, logfile: str): - client = docker.from_env() - container = client.containers.list(filters=dict(name=container_name))[0] - - print(container) - - def log_output(): - nonlocal container, logfile - print(f"Starting to write to file {logfile}.") - with open(logfile, 'wb') as fp: - print(f"openfile") - for p in container.logs(stream=True, stderr=PIPE, stdout=PIPE): - fp.write(p) - fp.flush() - - threading.Thread(target=log_output, daemon=True).start() - - - @contextlib.contextmanager - def run_containers(config, stop_after=True) -> Containers: - containers = Containers(config) - - containers.start() - try: - yield containers - finally: - if stop_after: - containers.stop() - - - @contextlib.contextmanager - def run_dependency_containers(stop_after=False) -> Containers: - - containers = Containers(DEFAULT_DOCKER_DEPENDENCY_CONFIG) - - containers.start() - try: - yield containers - finally: - if stop_after: - containers.stop() - - - @contextlib.contextmanager - def run_gridappsd_container(stop_after=True, rebuild_if_present=False) -> Container: - """ A contextmanager that uses """ - - parent_container = get_docker_in_docker() - - client = docker.from_env() - # if there is a parent_container then we need to make sure that it is connected - # to the same network as our systems. If not then we need to modify the network - # to include the parent container - if parent_container: - env = DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd'].get('environment') - if env is None: - env = {} - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.name] = 'gridappsd' - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.name] = 'test_app_user' - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.name] = '4Test' - DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd']['environment'] = env - - _log.debug(f"Running inside a container environment: {parent_container.name}") - network = client.networks.get(NETWORK) - has_it = False - for x in network.containers: - if x.name == parent_container.name: - has_it = True - _log.debug(f"parent_container {parent_container.name} is connected to the network.") - break - if not has_it: - _log.debug(f"Connecting new container to the network: {parent_container.name}") - network.connect(parent_container) - else: - _log.debug("Not running in a container") - - containers = Containers(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) - - gridappsd_container = None - try: - gridappsd_container = client.containers.get("gridappsd") - _log.info(f"{gridappsd_container.name} container found") - if rebuild_if_present: - gridappsd_container.kill() - except docker.errors.NotFound: - _log.debug("gridappsd container not found!") - - try: - if gridappsd_container is None: - containers.start() - - # the gridappsd container itself will take a bit to start up. - time.sleep(30) - - tries = 30 - while True: - tries -= 1 - if tries <=0: - raise RuntimeError("Couldn't connect to gridappsd server in a timely manner!") - try: - g = GridAPPSD() - if g.connected: - _log.info("Connected to gridappsd!") - g.disconnect() - break - - except stomp.exception.ConnectFailedException or stomp.exception.NotConnectedException: - _log.error("Retesting connection") - - yield gridappsd_container - finally: - if stop_after: - containers.stop() - - - def get_docker_in_docker() -> Container: - """ - Grab the parent container named the same as the current machine's hostname. We are assuming that this - is going to be a container. - """ - # There needs to be a test to make sure that the current container (assuming run in dev environment) - # is able to be run from the docker dev environment. That environment is going to be assumed to be - # the same name as the host name of the container. See the docker-compose starting the dev environment - hostname = str(open("/etc/hostname", "rt").read().strip()) - client = docker.from_env() - try: - parent_container = client.containers.get(hostname) - _log.info(f"Inside parent container: {hostname}") - # Setup to use gridappsd as the connection address. This value is used in the - # utils script to establish connection with the gridappsd server - os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" - except docker.errors.NotFound: - _log.debug(f"Docker container is not named this hostname {hostname}") - parent_container = None - return parent_container diff --git a/gridappsd/field_interface/agents/agents.py b/gridappsd/field_interface/agents/agents.py deleted file mode 100644 index 6436733..0000000 --- a/gridappsd/field_interface/agents/agents.py +++ /dev/null @@ -1,235 +0,0 @@ -from typing import Dict -import cimlab.data_profile.cimext_2022 as cim - -from abc import abstractmethod -from dataclasses import dataclass, field -import importlib -import logging - -from gridappsd.field_interface.context import ContextManager - -from cimlab.loaders import Parameter, ConnectionParameters -from cimlab.loaders.gridappsd import GridappsdConnection, get_topology_response -from cimlab.models import SwitchArea, SecondaryArea, DistributedModel - -from gridappsd.field_interface.gridappsd_field_bus import GridAPPSDMessageBus -from gridappsd.field_interface.interfaces import MessageBusDefinition - -import cimlab.data_profile.cimext_2022 as cim -from cimlab.loaders import Parameter, ConnectionParameters -from cimlab.loaders import gridappsd -from cimlab.loaders.gridappsd import GridappsdConnection, get_topology_response -from cimlab.models import SwitchArea, SecondaryArea, DistributedModel - - -cim = None -sparql = None - - -_log = logging.getLogger(__name__) - -def set_cim_profile(cim_profile): - global cim - cim = importlib.import_module('cimlab.data_profile.' + cim_profile) - gridappsd.set_cim_profile(cim_profile) - - -class DistributedAgent: - - - def __init__(self, - upstream_message_bus_def: MessageBusDefinition, - downstream_message_bus_def: MessageBusDefinition, - agent_dict=None, - simulation_id=None, - cim_profile: str = None): - """ - Creates a DistributedAgent object that connects to the specified message - buses and gets context based on feeder id and area id. - """ - _log.debug(f"Creating DistributedAgent: {self.__class__.__name__}") - self.upstream_message_bus = None - self.downstream_message_bus = None - self.simulation_id = simulation_id - self.context = None - self.params = ConnectionParameters() - self.connection = GridappsdConnection(self.params) - - if upstream_message_bus_def is not None: - if upstream_message_bus_def.is_ot_bus: - self.upstream_message_bus = GridAPPSDMessageBus(upstream_message_bus_def) - # else: - # self.upstream_message_bus = VolttronMessageBus(upstream_message_bus_def) - - if downstream_message_bus_def is not None: - if downstream_message_bus_def.is_ot_bus: - self.downstream_message_bus = GridAPPSDMessageBus(downstream_message_bus_def) - # else: - # self.downstream_message_bus = VolttronMessageBus(downstream_message_bus_def) - - # self.context = ContextManager.get(self.feeder_id, self.area_id) - - #if agent_dict is not None: - # self.addressable_equipments = agent_dict['addressable_equipment'] - # self.unaddressable_equipments = agent_dict['unaddressable_equipment'] - - @classmethod - def from_feeder(cls, feeder_id, area_id): - context = ContextManager.get_context_by_feeder(feeder_id, area_id) - return cls(context.get('upstream_message_bus_def'), context.get('downstream_message_bus_def')) - - def connect(self): - if self.upstream_message_bus is not None: - self.upstream_message_bus.connect() - if self.downstream_message_bus is not None: - self.downstream_message_bus.connect() - if self.downstream_message_bus is None and self.upstream_message_bus is None: - raise ValueError("Either upstream or downstream bus must be specified!") - self.subscribe_to_measurement() - - def subscribe_to_measurement(self): - if self.simulation_id is None: - self.downstream_message_bus.subscribe(f"fieldbus/{self.downstream_message_bus.id}", self.on_measurement) - else: - topic = f"/topic/goss.gridappsd.field.simulation.output.{self.simulation_id}.{self.downstream_message_bus.id}" - _log.debug(f"subscribing to sim_output on topic {topic}") - self.downstream_message_bus.subscribe(topic, - self.on_simulation_output) - - def on_measurement(self, headers: Dict, message) -> None: - raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") - - def on_simulation_output(self, headers, message): - self.on_measurement(headers=headers, message=message) - - -''' TODO this has not been implemented yet, so we are commented them out for now. - # not all agent would use this - def on_control(self, control): - device_id = control.get('device') - command = control.get('command') - self.control_device(device_id, command) - - def publish_to_upstream_bus(self,output): - self.switch_message_bus.publish(self.output_topic, output) - - # could be and upstream or peer level agent - def publish_to_upstream_bus_agent(self,agent_id, output): - self.switch_message_bus.publish(self.topic.agent_id, output) - - def publish_to_downstream_bus(self,message): - self.secondary_message_bus.publish(self.topic, message) - - # downstream agent - def publish_to_downstream_bus_agent(self,agent_id, message): - self.secondary_message_bus.publish(self.topic.agent_id, message) - - def control_device(self, device_id, command): - device_topic = self.devices.get(device_id) - self.secondary_message_bus.publish(device_topic, command)''' - - -class FeederAgent(DistributedAgent): - - def __init__(self, upstream_message_bus_def: MessageBusDefinition, - downstream_message_bus_def: MessageBusDefinition, - feeder_dict=None, simulation_id=None): - super(FeederAgent, self).__init__(upstream_message_bus_def, - downstream_message_bus_def, - feeder_dict, simulation_id) - - if feeder_dict is not None: - feeder = cim.Feeder(mRID=downstream_message_bus_def.id) - - self.feeder_area = DistributedModel(connection=self.connection, feeder=feeder, topology=feeder_dict) - - -class SwitchAreaAgent(DistributedAgent): - - def __init__(self, upstream_message_bus_def: MessageBusDefinition, - downstream_message_bus_def: MessageBusDefinition, - switch_area_dict=None, simulation_id=None): - - super().__init__(upstream_message_bus_def, - downstream_message_bus_def, - switch_area_dict, simulation_id) - - if switch_area_dict is not None: - self.switch_area = SwitchArea(downstream_message_bus_def.id, self.connection) - self.switch_area.initialize_switch_area(switch_area_dict) - - -class SecondaryAreaAgent(DistributedAgent): - - def __init__(self, upstream_message_bus_def: MessageBusDefinition, - downstream_message_bus_def: MessageBusDefinition, - secondary_area_dict=None, simulation_id=None): - - super().__init__(upstream_message_bus_def, - downstream_message_bus_def, - secondary_area_dict, simulation_id) - - if secondary_area_dict is not None: - self.secondary_area = SecondaryArea(downstream_message_bus_def.id, self.connection) - self.secondary_area.initialize_secondary_area(secondary_area_dict) - - -class CoordinatingAgent: - """ - A CoordinatingAgent performs following functions: - 1. Spawns distributed agents - 2. Publishes compiled output to centralized OT bus - 3. Distributes centralized output to Feeder bus and distributed agents - 4. May have connected devices and control those devices - - upstream, peer , downstream and broadcast - """ - - def __init__(self, feeder_id, system_message_bus_def: MessageBusDefinition, simulation_id=None): - self.feeder_id = feeder_id - self.distributed_agents = [] - - self.system_message_bus = GridAPPSDMessageBus(system_message_bus_def) - self.system_message_bus.connect() - # self.context = ContextManager.getContextByFeeder(self.feeder_id) - # print(self.context) - # self.addressable_equipments = self.context['data']['addressable_equipment'] - # self.unaddressable_equipments = self.context['data']['unaddressable_equipment'] - # self.switch_areas = self.context['data']['switch_areas'] - - # self.subscribe_to_feeder_bus() - - def spawn_distributed_agent(self, distributed_agent: DistributedAgent): - distributed_agent.connect() - self.distributed_agents.append(distributed_agent) - - -''' - def on_message_from_feeder_bus(self, message): - pass - - def subscribe_to_distribution_bus(self, topic): - #self.system_message_bus.subscribe("/topic/goss.gridappsd.field."+self.feeder_id, - self.on_message_from_feeder_bus) - self.system_message_bus.subscribe(topic, self.on_message_from_feeder_bus) - - def subscribe_to_feeder_bus(self, topic): - self.system_message_bus.subscribe(topic, self.on_message_from_feeder_bus) - - def on_measurement(self, measurements): - print(measurements) - - def on_control(self, control): - device_id = control.get('device') - command = control.get('command') - self.control_device(device_id, command) - - def publish_to_distribution_bus(self,message): - self.publish_to_downstream_bus(message) - - def publish_to_distribution_bus_agent(self,agent_id, message): - self.publish_to_downstream_bus_agent(agent_id, message) - - def control_device(self, device_id, command): - device_topic = self.devices.get(device_id) - self.secondary_message_bus.publish(device_topic, command)''' diff --git a/gridappsd/field_interface/context.py b/gridappsd/field_interface/context.py deleted file mode 100644 index ae1c882..0000000 --- a/gridappsd/field_interface/context.py +++ /dev/null @@ -1,36 +0,0 @@ -from gridappsd import GridAPPSD - -request_field_queue_prefix = 'goss.gridappsd.process.request.field' -request_field_context_queue = request_field_queue_prefix + '.context' - - -class ContextManager: - - @classmethod - def get_context_by_feeder(cls, feeder_mrid, area_id=None): - gridappsd_obj = GridAPPSD() - - request = {'modelId': feeder_mrid, - 'areaId': area_id} - - response = gridappsd_obj.get_response(request_field_context_queue, request) - return response - - @classmethod - def get_context_by_message_bus(cls, downstream_message_bus_id): - """ - return agents/devices based on upstream and/or downstream message bus as input - make message bus id a list - - based on filter return distributed agents for different applications as well - """ - gridappsd_obj = GridAPPSD() - - request = {'downstream_message_bus_id': downstream_message_bus_id, - 'agents': True, - 'devices': True} - - return gridappsd_obj.get_response(request_field_context_queue, request) - -# Provide context based on router (ip trace) or PKI -# Maybe able to emulate/simulate diff --git a/info/CHANGELOG.md b/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/info/VERSION b/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/pyproject.toml b/pyproject.toml index 0090974..f8a2099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] -name = "gridappsd-python" -version = "v2.7.230209" +name = "gridappsd-python-workspace" +version = "2023.5.2a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -16,34 +16,9 @@ keywords = ["gridappsd", "grid", "activmq", "powergrid", "simulation", "library" readme = "README.md" -include =["gridappsd/conf/*"] -packages = [ - { include = 'gridappsd'} -] - -[tool.poetry.scripts] -# Add things in the form -# myscript = 'my_package:main' -register_app = 'gridappsd.register_app:main' -gridappsd-cli = 'gridappsd.cli:_main' - - [tool.poetry.dependencies] python = ">=3.7.9,<4.0" -PyYAML = "^6.0" -pytz = "^2022.7" -dateutils = "^0.6.7" -#gridappsd-cim-profile={url="https://github.com/GRIDAPPSD/gridappsd-cim-profile/releases/download/v0.7.20230120180831a0/gridappsd_cim_profile-0.7.20230120180831a0-py3-none-any.whl"} -#gridappsd-cim-profile = "^0.10.20230208223046a0" -stomp-py = "6.0.0" -gridappsd-cim-lab = "^0.11.230209" -[tool.poetry.group.dev.dependencies] -pytest = "^6.2.2" -pytest-html = "^3.1.1" -mock = "^4.0.3" -docker = "^4.4.4" -yapf = "^0.32.0" [build-system] requires = ["poetry-core>=1.2.0"] diff --git a/scripts/create_local_version.sh b/scripts/create_local_version.sh new file mode 100755 index 0000000..8e4bcfe --- /dev/null +++ b/scripts/create_local_version.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Usus dunamai to determine a semver compatible version for the current state of the project +# Useefull when building wheels in CI/CD on branches or merge requests, +# without possibly overwriting released versions (of certain tag) +# Used to run in CI/CD, as it will modify both pyproject.toml's and python files (by setting the right string in `__version__=..`) +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# first run directly, to have script stop if dunamai isn't available (for example if not installed, or running in wrong virtual env) +dunamai from any +VERSION=$(dunamai from any) +echo $VERSION + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi +for p in $_projects +do + echo "Creating local version of ${p}" + echo "$VERSION" > "${p}/VERSION" + sed -i$SEP'' "s/^version = .*/version = \"$VERSION\"/" "$p/pyproject.toml" +done +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" package-a/package_a/__init__.py +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" package-b/package_b/__init__.py +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" service-c/service_c/__init__.py diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh new file mode 100755 index 0000000..3a909f5 --- /dev/null +++ b/scripts/poetry_build.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# This script builds all the poetry packages, creating wheels, dists, and requirements.txt's +# All the wheels will be placed in both the root folder's dist, and in a dist folder within each package +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +poetry version +VERSION=$(poetry version | awk '{print $2}') + +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=$PROJECTS +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + # change path deps in project def + sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml + # include project changelog + cp ../CHANGELOG.md ./ + poetry build + # export deps, with updated path deps + mkdir -p info + poetry export -f requirements.txt --output ./info/requirements.txt --without-hashes --with-credentials + sed -i$SEP'' "s/ @ .*;/==$VERSION;/" "./info/requirements.txt" + ls -altr ./dist/ +done + +# -u for update +if [ "$(uname)" = "Darwin" ]; then export FLAG=" "; else FLAG="-u "; fi +echo "==========" +mkdir -p "${DIR}/../info" +cp $FLAG "${DIR}/../CHANGELOG.md" "${DIR}/../info/" +cp $FLAG "${DIR}/../VERSION" "${DIR}/../info/" +echo "==========" +# copying each wheel to root folder dist +mkdir -p "${DIR}/../dist" +for p in $_projects +do + ls -altr "${DIR}/../${p}/dist/" + cp $FLAG "${DIR}/../${p}/dist/"*".whl" "${DIR}/../dist/" + cp $FLAG "${DIR}/../${p}/dist/"*".tar.gz" "${DIR}/../dist/" +done +echo "==========" +ls -altr "${DIR}/../dist/" +# then copying these to each project +# for p in $_projects +# do +# cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" +# cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" +# ls -altr "${DIR}/../${p}/dist/" +# done diff --git a/scripts/poetry_install.sh b/scripts/poetry_install.sh new file mode 100755 index 0000000..f4a0bc5 --- /dev/null +++ b/scripts/poetry_install.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# This script reflects the latest changes of pyproject.toml +# into both the poetry.lock file and the virtualenv. +# by running `poetry lock --no-update && poetry install --sync` +# It first configures poetry to use the right python for creation of the virtual env +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + poetry env use $(which python3) || poetry env use 3.8 + poetry lock --no-update && poetry install --sync +done diff --git a/scripts/poetry_update.sh b/scripts/poetry_update.sh new file mode 100755 index 0000000..4df08a9 --- /dev/null +++ b/scripts/poetry_update.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# This script reflects the latest changes of pyproject.toml +# into both the poetry.lock file and the virtualenv. +# by running `poetry update && poetry install --sync` +# It first configures poetry to use the right python for creation of the virtual env +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + poetry env use $(which python3) || poetry env use 3.8 + poetry update && poetry install --sync +done diff --git a/scripts/projects.sh b/scripts/projects.sh new file mode 100644 index 0000000..531e65c --- /dev/null +++ b/scripts/projects.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# all python packages, in topological order +PROJECTS='gridappsd-field-bus-lib gridappsd-python-lib' \ No newline at end of file diff --git a/scripts/replace_path_deps.sh b/scripts/replace_path_deps.sh new file mode 100755 index 0000000..0795803 --- /dev/null +++ b/scripts/replace_path_deps.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# This file shows how the sdist & wheel files can be manually modified afterwards +# to replace path dependencies with a version range. +# If the mono repo is at version 1.2.3, it will set the dependencies to (~=1.2,>=1.2.3), effectively equal to ~1.2.3. +set -x +set -u +set -e + +VERSION=$(poetry version | awk '{print $2}') +VERSION_MINOR=$(echo $VERSION | sed -E "s/^([0-9]*\.[0-9]*).*/\1/") +curdir=$(pwd) +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi + +# ===== Updating the TAR.GZ file ===== +cd "$curdir" +TARFILES=$(ls dist/*.tar.gz) +for TARFILE in $TARFILES +do + rm -rf /tmp/version_update + mkdir -p /tmp/version_update + tar -C /tmp/version_update -xf $curdir/$TARFILE + cd /tmp/version_update + # Replace the path dependencies (which are prefixed with '@') + # with compatible version to the current monorepo, but at least at the current one. + # In semver notation: ~1.2.3, which equals >=1.2.3, <2.0.0 + # Note that allowed matches are defined at: + # https://peps.python.org/pep-0440/#compatible-release + # We therefore specify that we require >=1.2.3 AND <2.0 + # Thus at least at the same fix version, but only compatible versions. + # Therefore we use ~=1.2, which equals >=1.2,<2.0, together with >=1.2.3 + FOLDER=$(ls) + sed -i$SEP'' "s|^Requires-Dist: \(.*\) @ \.\./.*|Requires-Dist: \1 (~=$VERSION_MINOR,>=$VERSION)|" "$FOLDER/PKG-INFO" + sed -i$SEP'' "s| @ \.\.[a-zA-Z\-_/]*|~=$VERSION_MINOR,>=$VERSION|" "$FOLDER/setup.py" + sed -i$SEP'' "s|{.*path.*\.\..*|\"~$VERSION\"|" "$FOLDER/pyproject.toml" + tar -czvf new.tar.gz "$FOLDER" + mv new.tar.gz $curdir/$TARFILE +done + +# ===== Updating the WHEEL file ===== +# Handle the tar.gz +cd "$curdir" +WHEELFILES=$(ls dist/*.whl) +for WHEELFILE in $WHEELFILES +do + rm -rf /tmp/version_update + mkdir -p /tmp/version_update + unzip -d /tmp/version_update $curdir/$WHEELFILE + cd /tmp/version_update + # Replace the path dependencies (which are prefixed with '@') + # with compatible version to the current monorepo, but at least at the current one. + # In semver notation: ~1.2.3, which equals >=1.2.3, <2.0.0 + # Note that allowed matches are defined at: + # https://peps.python.org/pep-0440/#compatible-release + # We therefore specify that we require >=1.2.3 AND <2.0 + # Thus at least at the same fix version, but only compatible versions. + # Therefore we use ~=1.2, which equals >=1.2,<2.0, together with >=1.2.3 + FOLDER=$(ls -d *.dist-info) + sed -i$SEP'' "s|^Requires-Dist: \(.*\) @ \.\./.*|Requires-Dist: \1 (~=$VERSION_MINOR,>=$VERSION)|" "$FOLDER/METADATA" + zip -r new.whl ./* + mv new.whl "$curdir/$WHEELFILE" + cd "$curdir" + rm -rf /tmp/version_update +done diff --git a/scripts/run_on_each.sh b/scripts/run_on_each.sh new file mode 100755 index 0000000..2722660 --- /dev/null +++ b/scripts/run_on_each.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# runs the passed command in each poetry project folder +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + echo "==running in ${p}==" + "$@" +done diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 231dc70..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,82 +0,0 @@ -import logging -import os -import sys - -import pytest - -from gridappsd import GridAPPSD, GOSS -from gridappsd.docker_handler import run_dependency_containers, run_gridappsd_container, Containers - -levels = dict( - CRITICAL=50, - FATAL=50, - ERROR=40, - WARNING=30, - WARN=30, - INFO=20, - DEBUG=10, - NOTSET=0 -) - -# Get string representation of the log level passed -LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") - -# Make sure the level passed is one of the valid levels. -if LOG_LEVEL not in levels.keys(): - raise AttributeError("Invalid LOG_LEVEL environmental variable set.") - -# Set the numeric version of log level to pass to the basicConfig function -LOG_LEVEL = levels[LOG_LEVEL] - -logging.basicConfig(stream=sys.stdout, level=LOG_LEVEL, - format="%(asctime)s|%(levelname)s|%(name)s|%(message)s") -logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) -logging.getLogger("docker.utils.config").setLevel(logging.INFO) -logging.getLogger("docker.auth").setLevel(logging.INFO) - - -STOP_CONTAINER_AFTER_TEST = os.environ.get('GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) -os.environ['GRIDAPPSD_USER'] = 'system' -os.environ['GRIDAPPSD_PASSWORD'] = 'manager' - - -@pytest.fixture(scope="module") -def docker_dependencies(): - print("Docker dependencies") - # Containers.reset_all_containers() - - with run_dependency_containers(stop_after=STOP_CONTAINER_AFTER_TEST) as dep: - yield dep - print("Cleanup docker dependencies") - - -@pytest.fixture -def gridappsd_client(request, docker_dependencies): - with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): - gappsd = GridAPPSD() - gappsd.connect() - assert gappsd.connected - models = gappsd.query_model_names() - assert models is not None - if request.cls is not None: - request.cls.gridappsd_client = gappsd - yield gappsd - - gappsd.disconnect() - - -@pytest.fixture -def goss_client(docker_dependencies): - with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): - goss = GOSS() - goss.connect() - assert goss.connected - - yield goss - - -@pytest.fixture -def foo(request): - if request.cls is not None: - request.cls.gridappsd_client = ["alpha", "beta", "gamma"] - diff --git a/tests/test_containers.py b/tests/test_containers.py deleted file mode 100644 index ae53276..0000000 --- a/tests/test_containers.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -import os -from pathlib import Path -import random -import shutil -import sys -from unittest import TestCase - - -_log = logging.getLogger("test_containers") - -try: - import docker - HAS_DOCKER = True -except ImportError: - _log.warning("Docker api not loaded. pip install docker to install as package.") - HAS_DOCKER = False - -from gridappsd.docker_handler import Containers - - -class ContainersTestCase(TestCase): - - @classmethod - def setUpClass(cls) -> None: - logging.basicConfig(level=logging.DEBUG,stream=sys.stdout) - logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) - logging.getLogger("docker.utils.config").setLevel(logging.INFO) - logging.getLogger("docker.auth").setLevel(logging.INFO) - cls.log = logging.getLogger("test_containers") - - cls.tmp_dir = Path("/tmp/tmpdir") - os.makedirs(cls.tmp_dir, exist_ok=True) - cls.tmp_file_name = cls.tmp_dir.joinpath("woot.txt") - cls.tmp_file_content = """ - here I come to save the day! -""" - with open(cls.tmp_file_name, "w") as stream: - stream.write(cls.tmp_file_content) - - def setUp(self) -> None: - self.cname = f"test_container_{random.randint(1,1000)}" - self.vname = f"test_volume_{random.randint(1,1000)}" - self.in_container_path = "/foo/bar/bim" - self.network_name = f"foo_{random.randint(1,1000)}" - self.client = docker.from_env() - - def test_can_create_volume(self): - # container_path = "/foo/bar/bim" - container = Containers.create_volume_container(name=self.cname, - volume_name=self.vname, - mount_in_container_at=self.in_container_path) - assert self.cname == container.name - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") - self.log.debug(f"Exit code: {exit_code}, result: {result}") - assert exit_code == 0 - - def test_can_copy_dir(self): - Containers.create_volume_container(name=self.cname, - volume_name=self.vname, - mount_in_container_at=self.in_container_path) - - Containers.copy_to(self.tmp_dir, f"{self.cname}:/foo/bar/bim/tmpdir") - - container = self.client.containers.get(self.cname) - - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") - - assert exit_code == 0 - assert b"tmpdir" in result - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}/tmpdir") - assert exit_code == 0 - assert b"woot.txt" in result - - def test_network_creation(self): - network = Containers.create_get_network(self.network_name) - assert network is not None - assert self.network_name == network.name - - def tearDown(self) -> None: - try: - container = self.client.containers.get(self.cname) - container.stop() - except docker.errors.NotFound: - pass - - try: - volume = self.client.volumes.get(self.vname) - volume.remove() - except docker.errors.NotFound: - pass - - try: - network = self.client.networks.get(self.network_name) - network.remove() - except docker.errors.NotFound: - pass - self.log.debug("tearDown") - - @classmethod - def tearDownClass(cls) -> None: - shutil.rmtree(cls.tmp_dir, ignore_errors=True) - cls.log.debug("tearDownClass") diff --git a/tests/test_docker_handler.py b/tests/test_docker_handler.py deleted file mode 100644 index 60a91fb..0000000 --- a/tests/test_docker_handler.py +++ /dev/null @@ -1,185 +0,0 @@ -import inspect -import logging -import os -from pathlib import Path - -import docker -import sys -import time - -from gridappsd import GridAPPSD -from gridappsd.docker_handler import (run_dependency_containers, Containers, run_gridappsd_container, - stream_container_log_to_file, DEFAULT_DOCKER_DEPENDENCY_CONFIG, - mysql_setup, MYSQL_SCHEMA_INIT_DIR) - -logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) -_log = logging.getLogger(inspect.getmodulename(__file__)) - - -def test_log_container(docker_dependencies): - mypath = "/tmp/alphabetagamma.txt" - stream_container_log_to_file("influxdb", mypath) - time.sleep(5) - print("After call to stream") - assert os.path.exists(mypath) - with open(mypath, 'rb') as rf: - assert len(rf.readlines()) > 0 - - -def test_can_reset_all_containers(): - Containers.reset_all_containers() - assert not Containers.container_list() - - config = { - "redis": { - "start": True, - "image": "redis:3.2.11-alpine", - "pull": True, - "ports": {"6379/tcp": 6379}, - "environment": [], - "links": "", - "volumes": "", - "entrypoint": "redis-server --appendonly yes", - } - } - cont = Containers(config) - cont.start() - assert len(Containers.container_list()) == 1 - time.sleep(5) - Containers.reset_all_containers() - assert not Containers.container_list() - - -def test_can_dependencies_continue_after_context_manager(): - my_config = DEFAULT_DOCKER_DEPENDENCY_CONFIG.copy() - Containers.reset_all_containers() - - time.sleep(3) - my_dep_containers = None - with run_dependency_containers() as containers: - my_dep_containers = containers - time.sleep(10) - - real_containers = Containers.container_list() - for k in my_dep_containers.container_def.keys(): - found = False - for c in real_containers: - if c.name == k: - found = True - break - assert found, f"Couldn't find {k} container in list" - - Containers.reset_all_containers() - - -def test_create_volume_container(): - Containers.create_volume_container("test_volume", "test_volume", "/startup", restart_if_exists=True) - path = str(Path("gridappsd/conf").absolute()) - Containers.copy_to(path, "test_volume:/startup/conf") - client = docker.from_env() - result = client.containers.get("test_volume").exec_run("ls -l /startup") - assert True - - -def test_can_upload_files_to_container(): - Containers.reset_all_containers() - - client = docker.from_env() - client.images.pull("alpine") - test_container = client.containers.run(image="alpine", command="tail -f /dev/null", - detach=True, - name="test_upload_container", - remove=True) - # may take a few for image to be up - time.sleep(20) - conf_path = str(Path("gridappsd/conf").absolute()) - Containers.copy_to(conf_path, f"{test_container.name}:/conf") - results = test_container.exec_run("ls -l /conf") - for f in os.listdir(conf_path): - assert f in results.output.decode("utf-8"), f"{f} was not in /conf" - - -def test_multiple_runs_in_a_row_with_dependency_context_manager(): - - Containers.reset_all_containers() - - with run_dependency_containers(): - pass - - containers = [x for x in Containers.container_list() if "config" not in x.name] - assert len(containers) == 5 - - with run_gridappsd_container(): - timeout = 0 - gapps = None - - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - with run_gridappsd_container(): - timeout = 0 - gapps = None - time.sleep(10) - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - -def test_can_start_gridappsd_within_dependency_context_manager_all_cleanup(): - - Containers.reset_all_containers() - - with run_dependency_containers(True) as cont: - # True in this method will remove the containsers - with run_gridappsd_container(True) as dep_con: - # Default cleanup is true within run_gridappsd_container method - timeout = 0 - gapps = None - time.sleep(10) - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - # Filter out the two config containers that we start up for volume data. - containers = [x.name for x in Containers.container_list() if "config" not in x.name] - assert not len(containers) - - -def test_can_start_gridapps(): - Containers.reset_all_containers() - with run_dependency_containers() as cont: - with run_gridappsd_container() as cont2: - g = GridAPPSD() - assert g.connected - - -def test_mysql_setup(): - mysql_setup() - assert Path(MYSQL_SCHEMA_INIT_DIR).exists() - assert Path(MYSQL_SCHEMA_INIT_DIR).joinpath("gridappsd_mysql_dump.sql").is_file() - diff --git a/tests/test_goss.py b/tests/test_goss.py deleted file mode 100644 index 25aa984..0000000 --- a/tests/test_goss.py +++ /dev/null @@ -1,212 +0,0 @@ -import json -import logging -import os -import threading -from queue import Queue - -import mock -import pytest -from time import sleep - -from gridappsd import GOSS -from gridappsd.docker_handler import get_docker_in_docker -from gridappsd.goss import GRIDAPPSD_ENV_ENUM - -_log = logging.getLogger(__name__) - - -def test_auth_raises_error_no_username_password(docker_dependencies): - container = get_docker_in_docker() - mockdict = { - GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.value: '', - GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.value: '' - } - if container: - mockdict[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.value] = "gridappsd" - - with mock.patch.dict(os.environ, mockdict): - with pytest.raises(ValueError) as ex: - goss = GOSS() - - with pytest.raises(ValueError) as ex: - goss = GOSS(username="foo") - - with pytest.raises(ValueError) as ex: - goss = GOSS(password="bar") - - -def test_get_response(caplog, goss_client): - caplog.set_level(logging.DEBUG) - - def addem_callback(header, message): - print("Addem callback") - print("Threadid: {}".format(threading.current_thread().ident)) - - if isinstance(message, str): - item = json.loads(message) - else: - item = message - total = 0 - for x in item: - total += x - - reply_to = header['reply-to'] - response = dict(result=total) - print("Sending back topic: {topic} {response}".format(topic=reply_to, - response=response)) - goss_client.send(reply_to, json.dumps(response)) - - gen_sub = [] - - def generic_subscription(header, message): - gen_sub.append((header, message)) - - # Simulate an rpc call. - goss_client.subscribe("/addem", addem_callback) - - goss_client.subscribe("foo", generic_subscription) - - # id_before = id(goss_client._conn) - result = goss_client.get_response('/addem', [5, 6]) - assert result['result'] == 11 - # assert id_before == id(goss_client._conn) - - goss_client.send("foo", str(result['result'])) - - count = 0 - while True: - sleep(0.1) - count += 1 - if len(gen_sub) > 0 or count > 10: - break - - assert gen_sub - assert len(gen_sub) == 1 - assert len(gen_sub[0]) == 2 - assert result['result'] == 11 - - -def test_send_receive(goss_client): - message_queue = Queue() - - class MyListener(object): - def on_message(self, headers, message): - message_queue.put((headers, message)) - - listener = MyListener() - goss_client.subscribe('doah', listener) - goss_client.send('doah', "I am a foo") - sleep(0.5) - assert message_queue.qsize() == 1 - header, message = message_queue.get() - assert message == "I am a foo" - - -def test_callback_function(goss_client): - message_queue1 = Queue() - - def callback1(headers, message): - message_queue1.put((headers, message)) - - goss_client.subscribe('foo', callback1) - goss_client.send('foo', "I am a foo") - sleep(0.5) - assert message_queue1.qsize() == 1 - header, message = message_queue1.get() - assert message == "I am a foo" - - -def test_multi_subscriptions(goss_client): - message_queue1 = Queue() - message_queue2 = Queue() - - def callback1(headers, message): - print(f"mq1 {headers} {message}") - message_queue1.put((headers, message)) - - def callback2(headers, message): - print(f"mq2 {headers} {message}") - message_queue2.put((headers, message)) - - goss_client.subscribe('bim', callback1) - goss_client.subscribe('bar', callback2) - sleep(0.5) - goss_client.send('bim', "I am a foo") - goss_client.send('bar', "I am a bar") - sleep(0.5) - assert message_queue1.qsize() == 1 - assert message_queue2.qsize() == 1 - header, message = message_queue1.get() - assert message == "I am a foo" - header, message = message_queue2.get() - assert message == "I am a bar" - - -def test_multi_subscriptions_same_topic(goss_client): - # pytest.xfail("Multiple topics can't be subscribed to the same topic at present.") - - message_queue1 = Queue() - message_queue2 = Queue() - - def callback1(headers, message): - print(f"handling callback1 {message} ") - message_queue1.put((headers, message)) - - def callback2(headers, message): - print(f"handling callback2 {message} ") - message_queue2.put((headers, message)) - - indx1 = goss_client.subscribe('bim', callback1) - goss_client.subscribe('bim', callback2) - sleep(0.5) - goss_client.send('bim', "I am a foo") - goss_client.send('bim', "I am a bar") - sleep(0.5) - assert message_queue1.qsize() == 2 - assert message_queue2.qsize() == 2 - header, message = message_queue1.get() - assert message == "I am a foo" - header, message = message_queue1.get() - assert message == "I am a bar" - header, message = message_queue2.get() - assert message == "I am a foo" - header, message = message_queue2.get() - assert message == "I am a bar" - - -def test_response_class(goss_client): - message_queue = Queue() - - class SubListener: - def on_message(self, header, message): - message_queue.put((header, message)) - - goss_client.subscribe("/topic/bar", SubListener()) - sleep(0.5) - goss_client.send("/topic/bar", {"abc": "def"}) - - result = message_queue.get() - - print(result) - assert result - assert 2 == len(result) - - assert dict(abc="def") == result[1] - - -def test_replace_subscription(caplog, goss_client): - caplog.set_level(logging.DEBUG) - original_queue = Queue() - after_queue = Queue() - - def original_callback(headers, message): - original_queue.put((headers, message)) - - def after_callback(headers, message): - after_queue.put((headers, message)) - - goss_client.subscribe("woot", original_callback) - goss_client.send("woot", "This is a message") - sleep(0.5) - - assert original_queue.qsize() == 1 diff --git a/tests/test_gridappsd.py b/tests/test_gridappsd.py deleted file mode 100644 index 4301dc8..0000000 --- a/tests/test_gridappsd.py +++ /dev/null @@ -1,140 +0,0 @@ -import logging -import os -import xml.etree.ElementTree as ET -from time import sleep - -import mock - -from gridappsd import GridAPPSD, topics as t, ProcessStatusEnum - - -def test_get_model_info(gridappsd_client): - """ The expecation is that we will have multiple models that we can retrieve from the - database. Two of which should have the model name of ieee8500 and ieee123. The models - should have the correct entry keys. - """ - - gappsd = gridappsd_client - import time - time.sleep(10) - info = gappsd.query_model_info() - - node_8500 = None - node_123 = None - for info_def in info['data']['models']: - if info_def['modelName'] == 'ieee8500': - node_8500 = info_def - elif info_def['modelName'] == 'ieee123': - node_123 = info_def - - assert node_123, "Missing the 123 model" - assert node_8500, "Missing 8500 node model." - - keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", - "regionName", "regionId"] - correct_keys = set(keys) - - assert len(correct_keys) == len(node_123) - assert len(correct_keys) == len(node_8500) - - for x in node_123: - correct_keys.remove(x) - - assert len(correct_keys) == 0 - - correct_keys = set(keys) - - for x in node_8500: - correct_keys.remove(x) - - assert len(correct_keys) == 0 - - -def test_listener_multi_topic(gridappsd_client): - gappsd = gridappsd_client - - class Listener: - def __init__(self): - self.call_count = 0 - - def reset(self): - self.call_count = 0 - - def on_message(self, headers, message): - print("Message was: {}".format(message)) - self.call_count += 1 - - listener = Listener() - - input_topic = t.simulation_input_topic("5144") - output_topic = t.simulation_output_topic("5144") - - gappsd.subscribe(input_topic, listener) - gappsd.subscribe(output_topic, listener) - - gappsd.send(input_topic, "Any message") - sleep(1) - assert 1 == listener.call_count - listener.reset() - gappsd.send(output_topic, "No big deal") - sleep(1) - assert 1 == listener.call_count - - -@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", - "GRIDAPPSD_SIMULATION_ID": "1234"}) -def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): - - class Listener: - def __init__(self): - self.call_count = 0 - - def reset(self): - self.call_count = 0 - - def on_message(self, headers, message): - print("Message was: {}".format(message)) - self.call_count += 1 - - listener = Listener() - gappsd = gridappsd_client - assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' - assert gappsd.get_simulation_id() == "1234" - - log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) - gappsd.subscribe(log_topic, listener) - gappsd.send_simulation_status("RUNNING", - "testing the sending and recieving of send_simulation_status().", - logging.DEBUG) - sleep(1) - assert listener.call_count == 1 - - new_log_topic = t.simulation_log_topic("54232") - gappsd.set_simulation_id(54232) - gappsd.subscribe(new_log_topic, listener) - gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") - sleep(1) - assert listener.call_count == 2 - - - -@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) -def test_gridappsd_status(gridappsd_client): - gappsd = gridappsd_client - assert "helics_goss_bridge.py" == gappsd.get_application_id() - assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value - assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value - gappsd.set_application_status("RUNNING") - - assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value - assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value - - gappsd.set_service_status("COMPLETE") - assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value - assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - - # Invalid - gappsd.set_service_status("Foo") - assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value - assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - diff --git a/tests/test_logging.py b/tests/test_logging.py deleted file mode 100644 index 50e3556..0000000 --- a/tests/test_logging.py +++ /dev/null @@ -1,122 +0,0 @@ -from gridappsd.loghandler import Logger -from gridappsd import topics as t, ProcessStatusEnum -import os -import mock -from mock import Mock -import pytest - - -def init_gapps_mock(simulation_id=None, application_id=None, process_status=None, service_id=None): - gapps = Mock() - - gapps.get_simulation_id.return_value = simulation_id - gapps.get_application_id.return_value = application_id - gapps.get_application_status.return_value = process_status - gapps.get_process_id.return_value = service_id - - return gapps - - -#@mock.patch('gridappsd.utils.get_application_id') -def test_required_application_id_set(): - """ os.environ['GRIDAPPSD_APPLICATION_ID'] must be set to run.""" - log = Logger(init_gapps_mock()) - - with pytest.raises(AttributeError): - log.debug("foo") - - -def test_no_simulation_id_topic_or_application_id(): - """If no simulation then the topic should be the platform log topic""" - expected_topic = t.platform_log_topic() - - gapps_mock = init_gapps_mock(application_id="my_app_id", - process_status=ProcessStatusEnum.STARTING.value) - log = Logger(gapps_mock) - - log.debug("A message") - - topic, message = gapps_mock.send.call_args.args - - assert expected_topic == topic - assert message['processStatus'] == ProcessStatusEnum.STARTING.value - assert message['logMessage'] == 'A message' - - -def test_platform_log(): - - application_id = "my_app" - gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) - log = Logger(gapps_mock) - - log.debug("foo") - gapps_mock.send.assert_called_once() - - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - - assert message['source'] == application_id - assert message['logLevel'] == 'DEBUG' - assert message['logMessage'] == 'foo' - assert message['processStatus'] == ProcessStatusEnum.STOPPING.value - gapps_mock.send.reset_mock() - - log.info("bar") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'INFO' == message['logLevel'] - assert 'bar' == message['logMessage'] - gapps_mock.send.reset_mock() - - log.error("bim") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'ERROR' == message['logLevel'] - assert 'bim' == message['logMessage'] - gapps_mock.send.reset_mock() - - log.warning("baf") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'WARN' == message['logLevel'] - assert 'baf' == message['logMessage'] - - -def test_invalid_log_level(): - application_id = "my_app" - gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) - log = Logger(gapps_mock) - - with pytest.raises(AttributeError): - log.log("junk error", "BART") - - -def test_topic_and_status_set_correctly(): - - sim_id = "543" - application_id = "wicked_good_app_id" - mock_gapps = init_gapps_mock(simulation_id=sim_id, application_id=application_id, - process_status=ProcessStatusEnum.RUNNING.value) - - expected_topic = t.simulation_log_topic(sim_id) - - log = Logger(mock_gapps) - - log.debug("A message") - - # During the call to debug we expect the send function to be - # called on the mock object. Grab the arguments and then - # make sure that they are what we expect. - topic, message = mock_gapps.send.call_args.args - - assert message['source'] == application_id - assert topic == expected_topic - assert message['processStatus'] == "RUNNING" - - diff --git a/tests/test_logging_integration.py b/tests/test_logging_integration.py deleted file mode 100644 index e4aa613..0000000 --- a/tests/test_logging_integration.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import time - -import mock -import pytest - -from gridappsd import GridAPPSD, topics as t -from gridappsd.loghandler import Logger - - -@pytest.fixture -def logger_and_gridapspd(gridappsd_client) -> (Logger, GridAPPSD): - - logger = Logger(gridappsd_client) - - yield logger, gridappsd_client - - logger = None - - -@mock.patch.dict(os.environ, - dict(GRIDAPPSD_APPLICATION_ID='sample_app', - GRIDAPPSD_APPLICATION_STATUS='RUNNING')) -def test_log_stored(logger_and_gridapspd): - logger, gapps = logger_and_gridapspd - - log_data_map = [ - (logger.debug, "A debug message", "DEBUG"), - (logger.info, "An info message", "INFO"), - (logger.error, "An error message", "ERROR"), - (logger.error, "Another error message", "ERROR"), - (logger.info, "Another info message", "INFO"), - (logger.debug, "A debug message", "DEBUG") - ] - - assert gapps.connected - - # Make the calls to debug - for d in log_data_map: - d[0](d[1]) - - payload = { - "query": "select * from log order by timestamp" - } - time.sleep(5) - response = gapps.get_response(t.LOGS, payload, timeout=60) - assert response['data'], "There were not any records returned." - - for x in response['data']: - if x['source'] != 'sample_app': - continue - expected = log_data_map.pop(0) - assert expected[1] == x['log_message'] - assert expected[2] == x['log_level'] - - -SIMULATION_ID='54321' - - #TODO Ask about loging api for simulations. -@mock.patch.dict(os.environ, - dict(GRIDAPPSD_APPLICATION_ID='new_sample_app', - GRIDAPPSD_APPLICATION_STATUS='RUNNING', - GRIDAPPSD_SIMULATION_ID=SIMULATION_ID)) -def test_simulation_log_stored(logger_and_gridapspd): - logger, gapps = logger_and_gridapspd - - assert gapps.get_simulation_id() == SIMULATION_ID - - log_data_map = [ - (logger.debug, "A debug message", "DEBUG"), - (logger.info, "An info message", "INFO"), - (logger.error, "An error message", "ERROR"), - (logger.error, "Another error message", "ERROR"), - (logger.info, "Another info message", "INFO"), - (logger.debug, "A debug message", "DEBUG") - ] - - assert gapps.connected - - # Make the calls to debug - for d in log_data_map: - d[0](d[1]) - - time.sleep(5) - payload = { - "query": "select * from log" - } - - response = gapps.get_response(t.LOGS, payload, timeout=60) - assert response['data'], "There were not any records returned." - - for x in response['data']: - if x['source'] != 'new_sample_app': - continue - expected = log_data_map.pop(0) - assert expected[1] == x['log_message'] - assert expected[2] == x['log_level'] diff --git a/tests/test_simulation.py b/tests/test_simulation.py deleted file mode 100644 index b4c85d2..0000000 --- a/tests/test_simulation.py +++ /dev/null @@ -1,45 +0,0 @@ -import json -# from pprint import pprint -import logging -import os -import sys -import time -import pytest - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -from gridappsd import GridAPPSD, topics as t -from gridappsd.simulation import Simulation - -# The directory containing this file -HERE = os.path.dirname(__file__) - - -def base_config(): - data = {"power_system_config":{"SubGeographicalRegion_name":"_ABEB635F-729D-24BF-B8A4-E2EF268D8B9E","GeographicalRegion_name":"_73C512BD-7249-4F50-50DA-D93849B89C43","Line_name":"_49AD8E07-3BF9-A4E2-CB8F-C3722F837B62"},"simulation_config":{"power_flow_solver_method":"NR","duration":120,"simulation_name":"ieee13nodeckt","simulator":"GridLAB-D","start_time":1605418946,"run_realtime":False,"simulation_output":{},"model_creation_config":{"load_scaling_factor":1.0,"triplex":"y","encoding":"u","system_frequency":60,"voltage_multiplier":1.0,"power_unit_conversion":1.0,"unique_names":"y","schedule_name":"ieeezipload","z_fraction":0.0,"i_fraction":1.0,"p_fraction":0.0,"randomize_zipload_fractions":False,"use_houses":False},"simulation_broker_port":51044,"simulation_broker_location":"127.0.0.1"},"application_config":{"applications":[]},"service_configs":[],"test_config":{"randomNum":{"seed":{"value":185213303967438},"nextNextGaussian":0.0,"haveNextNextGaussian":False},"events":[],"testInput":True,"testOutput":True,"appId":"","testId":"1468836560","testType":"simulation_vs_expected","storeMatches":False},"simulation_request_type":"NEW"} - # with open("{HERE}/simulation_fixtures/13_node_2_min_base.json".format(HERE=HERE)) as fp: - # data = json.load(fp) - return data - - -def test_simulation_no_duplicate_measurement_timestamps(gridappsd_client: GridAPPSD): - num_measurements = 0 - timestamps = set() - - def measurement(sim, timestamp, measurement): - nonlocal num_measurements - num_measurements += 1 - assert timestamp not in timestamps - timestamps.add(timestamp) - - gapps = gridappsd_client - sim = Simulation(gapps, base_config()) - sim.add_onmeasurement_callback(measurement) - sim.start_simulation() - sim.run_loop() - - # did we get a measurement? - assert num_measurements > 0 - - # if empty then we know the simulation did not work. - assert timestamps