From c305538ae675c0c83118748144c75136d3d6b454 Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Wed, 4 Oct 2023 23:48:43 +0200 Subject: [PATCH 1/4] changed README and Dockerfile --- Dockerfile | 7 ++----- README.md | 49 ++++++++++++++++++++++++------------------------ requirements.txt | 30 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 30 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 5349c94c..e94325fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM python:3.10-slim-buster -RUN pip install pipenv WORKDIR /yacg @@ -14,17 +13,15 @@ ADD randomDataServer.py randomDataServer.py ADD customRandomConstraints.py customRandomConstraints.py ADD validate.py validate.py ADD version.txt version.txt -ADD Pipfile Pipfile -ADD Pipfile.lock Pipfile.lock +ADD requirements.txt requirements.txt COPY yacg yacg/ #RUN cd /yacg && \ # pipenv --python 3.8 && \ # pipenv --three install --system -RUN pipenv requirements > requirements.txt RUN pip install -r requirements.txt WORKDIR /yacg -ENTRYPOINT [ "python3", "yacg.py"] \ No newline at end of file +ENTRYPOINT [ "python3", "yacg.py"] diff --git a/README.md b/README.md index a366af59..1c57fedf 100644 --- a/README.md +++ b/README.md @@ -46,38 +46,37 @@ ln -s /bin/pip ```bash # basic preparation sudo apt-get install python3-venv -pip install --user pipenv -pipenv --python 3.11 -pipenv install -pipenv shell +virtualenv venv +source venv/bin/activate + +pip install -r requirements.txt + -# in case of errors to create virtual env look here -# https://askubuntu.com/questions/1241993/why-i-cannot-create-pipenv-shell-in-ubuntu-20-04-lts-with-python3-8 # do a demo run ... and create plantuml -pipenv run python3 yacg.py \ +python yacg.py \ --models resources/models/json/yacg_config_schema.json \ resources/models/json/yacg_model_schema.json \ --singleFileTemplates plantUml=stdout -pipenv run python3 yacg.py \ +python yacg.py \ --models resources/models/json/yacg_config_schema.json \ resources/models/json/yacg_model_schema.json \ --usedFilesOnly # demo run with protobuf example output -pipenv run python3 yacg.py \ +python yacg.py \ --models resources/models/json/yacg_config_schema.json \ resources/models/json/yacg_model_schema.json \ --singleFileTemplates protobuf=stdout # run a test -pipenv run python3 -m unittest -v tests/model/test_model.py +python -m unittest -v tests/model/test_model.py # run all tests -pipenv run python3 -m unittest discover tests "test_*.py" +python -m unittest discover tests "test_*.py" ``` ## Docker ```bash @@ -118,10 +117,10 @@ This project contains also a script to increment model versions. I has the abili ## Usage ```bash # see the possible parameters of the script -pipenv run python3 incrementVersion.py +python incrementVersion.py # do an example dry-run -pipenv run python3 incrementVersion.py --model resources/models/json/yacg_model_schema.json --version minor +python incrementVersion.py --model resources/models/json/yacg_model_schema.json --version minor ``` @@ -131,20 +130,20 @@ This project contains also a script to convert JSON schemas to the yaml format. ## Usage ```bash # see the possible parameters of the script -pipenv run python3 modelToYaml.py --help +python modelToYaml.py --help # do an example dry-run -pipenv run python3 modelToYaml.py --model resources/models/json/yacg_model_schema.json --dryRun +python modelToYaml.py --model resources/models/json/yacg_model_schema.json --dryRun # feed stdin to convert -cat resources/models/json/yacg_model_schema.json | pipenv run python3 modelToYaml.py --stdin --dryRun +cat resources/models/json/yacg_model_schema.json | python modelToYaml.py --stdin --dryRun # more sophisticated example to create openApi yaml -pipenv run python3 yacg.py \ +python yacg.py \ --models tests/resources/models/json/examples/openapi_v3_example_refs.json \ --singleFileTemplates resources/templates/examples/normalizedOpenApiJson.mako=/tmp/test.json && \ - pipenv run python3 modelToYaml.py --model /tmp/test.json --destDir /tmp + python modelToYaml.py --model /tmp/test.json --destDir /tmp ``` # Models to JSON @@ -153,13 +152,13 @@ This project contains also a script to convert yaml schemas to the JSON format. ## Usage ```bash # see the possible parameters of the script -pipenv run python3 modelToJson.py --help +python modelToJson.py --help # do an example dry-run -pipenv run python3 modelToJson.py --model resources/models/yaml/yacg_config_schema.yaml --dryRun +python modelToJson.py --model resources/models/yaml/yacg_config_schema.yaml --dryRun # feed stdin to convert -cat resources/models/yaml/yacg_config_schema.yaml | pipenv run python3 modelToJson.py \ +cat resources/models/yaml/yacg_config_schema.yaml | python modelToJson.py \ --stdin --dryRun ``` @@ -170,7 +169,7 @@ schema ## Usage ```bash # python version -pipenv run python3 normalizeSchema.py --model +python normalizeSchema.py --model # docker version docker run -u $(id -u ${USER}):$(id -g ${USER}) \ @@ -193,7 +192,7 @@ from the schema input and tries the validation without it. ## Usage ```bash # bash version -pipenv run python3 validate.py --schema resources/models/json/yacg_config_schema.json \ +python validate.py --schema resources/models/json/yacg_config_schema.json \ --inputFile resources/configurations/conf_with_vars.json \ --draft07hack # used because json-spec has currently not draft07 support @@ -215,7 +214,7 @@ For further configurations see [here](docs/random_data_creation.md) ```bash # creates in the tmp folder a file Job.json with 10 random objects of the Job type -pipenv run python3 createRandomData.py --model resources/models/json/yacg_config_schema.json \ +python createRandomData.py --model resources/models/json/yacg_config_schema.json \ --type Job \ --defaultElemCount 10 --outputDir ./tmp @@ -229,7 +228,7 @@ with that script. ```bash # will start a simple http server on port 8080 that provides random data for a given type -pipenv run python3 randomDataServer.py --model resources/models/json/yacg_config_schema.json +python randomDataServer.py --model resources/models/json/yacg_config_schema.json # reading one random data set of the type 'Job' curl 'http://localhost:8080/job' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..42d653a3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +certifi==2023.7.22 +charset-normalizer==3.3.0 +colorama==0.4.6 +Faker==18.4.0 +flake8==4.0.1 +idna==3.4 +importlib-metadata==5.2.0 +jedi==0.19.1 +json-spec==0.11.0 +Mako==1.2.4 +MarkupSafe==2.1.3 +mccabe==0.6.1 +packaging==23.2 +parso==0.8.3 +ptvsd==4.3.2 +pudb==2022.1.3 +pycodestyle==2.8.0 +pyflakes==2.4.0 +Pygments==2.16.1 +pyparsing==3.0.9 +python-dateutil==2.8.2 +PyYAML==6.0 +requests==2.31.0 +semver==3.0.0 +shell==1.0.1 +six==1.16.0 +urllib3==2.0.6 +urwid==2.2.2 +urwid-readline==0.13 +zipp==3.17.0 From 8ab5a7e60c2b6f4b026673da27abe9da3e592c74 Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Wed, 4 Oct 2023 23:59:29 +0200 Subject: [PATCH 2/4] remove more pipenvs --- bin/demoGenerateCodeForYacg_configFile.sh | 4 ++-- bin/demoMultiFileGenerator.sh | 2 +- bin/generateCodeForYacg.sh | 14 +++++++------- bin/generateNormalizedOpenApi_example.sh | 6 +++--- bin/runGithubActions.sh | 22 +++++++++++----------- tests/generators/test_javaFuncs.py | 2 +- tests/model/test_modelFunc.py | 2 +- tests/model/test_templateHelper.py | 2 +- tests/util/test_additional.py | 2 +- tests/util/test_stringUtils.py | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/bin/demoGenerateCodeForYacg_configFile.sh b/bin/demoGenerateCodeForYacg_configFile.sh index 16ce67bb..204099de 100755 --- a/bin/demoGenerateCodeForYacg_configFile.sh +++ b/bin/demoGenerateCodeForYacg_configFile.sh @@ -8,9 +8,9 @@ scriptPos=${0%/*} pushd $scriptPos/.. > /dev/null echo "create yacg demo code based on a config file ..." -if ! pipenv run python3 yacg.py --config resources/configurations/gen_yacg_code.json; then +if ! python yacg.py --config resources/configurations/gen_yacg_code.json; then echo " ERROR while create the stuff :-/" exit 1 fi -popd > /dev/null \ No newline at end of file +popd > /dev/null diff --git a/bin/demoMultiFileGenerator.sh b/bin/demoMultiFileGenerator.sh index a9eff188..95ba7e62 100755 --- a/bin/demoMultiFileGenerator.sh +++ b/bin/demoMultiFileGenerator.sh @@ -8,7 +8,7 @@ scriptPos=${0%/*} pushd $scriptPos/.. > /dev/null echo "create meta model cconfiglasses ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_model_schema.json \ --multiFileTemplates ${scriptPos}/../resources/templates/examples/javaBeans.mako=${scriptPos}/../tmp/cmdJavaBeans/de/test/model \ --templateParameters modelPackage=de.test.model \ diff --git a/bin/generateCodeForYacg.sh b/bin/generateCodeForYacg.sh index 65ee5d84..96a08133 100755 --- a/bin/generateCodeForYacg.sh +++ b/bin/generateCodeForYacg.sh @@ -11,7 +11,7 @@ scriptPos=${0%/*} pushd $scriptPos/.. > /dev/null echo "create meta model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_model_schema.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/model.py \ pythonBeansTests=${scriptPos}/../tests/model/test_model.py \ @@ -25,7 +25,7 @@ if ! pipenv run python3 yacg.py --models \ fi echo "create config model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_config_schema.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/config.py \ pythonBeansTests=${scriptPos}/../tests/model/test_config.py \ @@ -39,7 +39,7 @@ if ! pipenv run python3 yacg.py --models \ fi echo "create random config model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_random_data_types.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/random_config.py \ pythonBeansTests=${scriptPos}/../tests/model/test_random_config.py \ @@ -53,7 +53,7 @@ if ! pipenv run python3 yacg.py --models \ fi echo "create shared model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/shared/info.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/shared/info.py \ --protocolFile logs/gen_shared_info.log \ @@ -66,7 +66,7 @@ fi echo "create openapi model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_openapi_paths.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/openapi.py \ pythonBeansTests=${scriptPos}/../tests/model/test_openapi.py \ @@ -81,7 +81,7 @@ if ! pipenv run python3 yacg.py --models \ fi echo "create asyncapi model classes ..." -if ! pipenv run python3 yacg.py --models \ +if ! python yacg.py --models \ resources/models/json/yacg_asyncapi_types.json \ --singleFileTemplates pythonBeans=${scriptPos}/../yacg/model/asyncapi.py \ pythonBeansTests=${scriptPos}/../tests/model/test_asyncapi.py \ @@ -96,4 +96,4 @@ if ! pipenv run python3 yacg.py --models \ fi -popd > /dev/null \ No newline at end of file +popd > /dev/null diff --git a/bin/generateNormalizedOpenApi_example.sh b/bin/generateNormalizedOpenApi_example.sh index 869847e3..f280b0dc 100755 --- a/bin/generateNormalizedOpenApi_example.sh +++ b/bin/generateNormalizedOpenApi_example.sh @@ -7,14 +7,14 @@ scriptPos=${0%/*} pushd $scriptPos/.. > /dev/null -if ! pipenv run python3 yacg.py \ +if ! python yacg.py \ --noLogs \ --models tests/resources/models/json/examples/openapi_v3_example_refs.json \ --singleFileTemplates resources/templates/examples/normalizedOpenApiJson.mako=stdout \ - | pipenv run python3 modelToYaml.py --stdin --dryRun + | python modelToYaml.py --stdin --dryRun then echo " ERROR while create the stuff :-/" exit 1 fi -popd > /dev/null \ No newline at end of file +popd > /dev/null diff --git a/bin/runGithubActions.sh b/bin/runGithubActions.sh index 3e559ae2..56e0fea8 100755 --- a/bin/runGithubActions.sh +++ b/bin/runGithubActions.sh @@ -20,13 +20,13 @@ if ! pipenv run flake8 . --count --exit-zero --max-complexity=15 --max-line-leng exit 1 fi -if ! pipenv run python3 -m unittest discover tests "test_*.py"; then +if ! python -m unittest discover tests "test_*.py"; then echo "There are problems in the tests" popd > /dev/null exit 1 fi -if ! pipenv run python3 yacg.py \ +if ! python yacg.py \ --models resources/models/json/yacg_config_schema.json \ resources/models/json/yacg_model_schema.json \ --singleFileTemplates plantUml=stdout > /dev/null; then @@ -35,7 +35,7 @@ if ! pipenv run python3 yacg.py \ exit 1 fi -if ! pipenv run python3 yacg.py \ +if ! python yacg.py \ --models resources/models/json/yacg_config_schema.json \ resources/models/json/yacg_model_schema.json \ --usedFilesOnly > /dev/null; then @@ -51,21 +51,21 @@ if ! bin/demoMultiFileGenerator.sh > /dev/null; then exit 1 fi -if ! pipenv run python3 \ +if ! python \ modelToYaml.py --model resources/models/json/yacg_model_schema.json --dryRun &> /dev/null; then echo "problems while run modelToYaml.py" popd > /dev/null exit 1 fi -if ! pipenv run python3 \ +if ! python \ modelToJson.py --model resources/models/yaml/yacg_config_schema.yaml --dryRun &> /dev/null; then echo "problems while run modelToJson.py" popd > /dev/null exit 1 fi -if ! pipenv run python3 \ +if ! python \ validate.py --schema resources/models/json/yacg_config_schema.json \ --inputFile resources/configurations/conf_with_vars.json --draft07hack &> /dev/null; then echo "can't validate schema" @@ -73,7 +73,7 @@ if ! pipenv run python3 \ exit 1 fi -if pipenv run python3 \ +if python \ validate.py --schema resources/models/json/yacg_config_schema.json \ --inputFile resources/configurations/conf_with_vars.json &> /dev/null; then echo "wrong validation of schema" @@ -81,7 +81,7 @@ if pipenv run python3 \ exit 1 fi -if ! pipenv run python3 createRandomData.py --model resources/models/json/yacg_config_schema.json \ +if ! python createRandomData.py --model resources/models/json/yacg_config_schema.json \ --type Job \ --defaultElemCount 10 \ --outputDir ./tmp &> /dev/null; then @@ -90,7 +90,7 @@ if ! pipenv run python3 createRandomData.py --model resources/models/json/yacg_c exit 1 fi -if ! pipenv run python3 yacg.py \ +if ! python yacg.py \ --models https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/models/model_0202.json \ --delExistingStoredTemplates \ --singleFileTemplates https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/codegen/templates/golang_types.mako=stdout > /dev/null; then @@ -99,7 +99,7 @@ if ! pipenv run python3 yacg.py \ exit 1 fi -if pipenv run python3 yacg.py \ +if python yacg.py \ --models https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/models/model_0203.json \ --delExistingStoredTemplates \ --singleFileTemplates https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/codegen/templates/golang_types.mako=stdout; then @@ -108,7 +108,7 @@ if pipenv run python3 yacg.py \ exit 1 fi -if pipenv run python3 yacg.py \ +if python yacg.py \ --models https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/models/model_0202.json \ --delExistingStoredTemplates \ --singleFileTemplates https://raw.githubusercontent.com/OkieOth/oth.types/master/configs/codegen/templates/golang_types_X.mako=stdout; then diff --git a/tests/generators/test_javaFuncs.py b/tests/generators/test_javaFuncs.py index d6e2c350..fb38dc4d 100644 --- a/tests/generators/test_javaFuncs.py +++ b/tests/generators/test_javaFuncs.py @@ -21,7 +21,7 @@ def getExampleType(): return modelTypes[0] -# For executing these tests run: pipenv run python3 -m unittest -v tests/generators/test_javaFuncs.py +# For executing these tests run: python -m unittest -v tests/generators/test_javaFuncs.py class TestJavaFuncs (unittest.TestCase): def testIsDouble(self): diff --git a/tests/model/test_modelFunc.py b/tests/model/test_modelFunc.py index 439127dc..b7d19ada 100644 --- a/tests/model/test_modelFunc.py +++ b/tests/model/test_modelFunc.py @@ -46,7 +46,7 @@ def getComplexTypeWithBase(): return myType -# For executing these test run: pipenv run python3 -m unittest -v tests/model/test_modelFunc.py +# For executing these test run: python -m unittest -v tests/model/test_modelFunc.py class TestModelFuncs (unittest.TestCase): def testHasPropertyOfTypeNoBaseType(self): diff --git a/tests/model/test_templateHelper.py b/tests/model/test_templateHelper.py index 56e2cb5f..16b740fd 100644 --- a/tests/model/test_templateHelper.py +++ b/tests/model/test_templateHelper.py @@ -3,7 +3,7 @@ import yacg.templateHelper as templateHelper -# For executing these test run: pipenv run python3 -m unittest -v tests/model/test_templateHelper.py +# For executing these test run: python -m unittest -v tests/model/test_templateHelper.py class TestTemplateHelper (unittest.TestCase): def testAddLineBreakToDesc(self): diff --git a/tests/util/test_additional.py b/tests/util/test_additional.py index 4285d03a..c7a9d050 100644 --- a/tests/util/test_additional.py +++ b/tests/util/test_additional.py @@ -5,7 +5,7 @@ import yacg.model.modelFuncs as modelFuncs -# For executing these tests run: pipenv run python3 -m unittest -v tests/util/test_additional.py +# For executing these tests run: python -m unittest -v tests/util/test_additional.py class TestAdditional (unittest.TestCase): def _estFlattenTypes(self): modelFile = 'tests/resources/models/json/examples/more_sophisticated_allof.json' diff --git a/tests/util/test_stringUtils.py b/tests/util/test_stringUtils.py index df6186e7..26ef55f2 100644 --- a/tests/util/test_stringUtils.py +++ b/tests/util/test_stringUtils.py @@ -2,7 +2,7 @@ from yacg.util.stringUtils import toUpperCamelCase, toName from yacg.util.stringUtils import snakeToLowerCamelCase, snakeToUpperCamelCase, toLowerCase, toUpperCase -# For executing these test run: pipenv run python3 -m unittest -v tests/util/test_stringUtils.py +# For executing these test run: python -m unittest -v tests/util/test_stringUtils.py class TestStringUtils (unittest.TestCase): def testUpperCamelCase(self): t = toUpperCamelCase('i am a test') From 7eb6f46f8b408fff8abb524848d729c799170a9b Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Thu, 16 Nov 2023 13:24:04 +0100 Subject: [PATCH 3/4] fixed github actions --- bin/runGithubActions.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/runGithubActions.sh b/bin/runGithubActions.sh index 56e0fea8..804cefc7 100755 --- a/bin/runGithubActions.sh +++ b/bin/runGithubActions.sh @@ -4,20 +4,28 @@ scriptPos=${0%/*} pushd $scriptPos/.. > /dev/null +echo "remove content of tmp folder ..." + rm -rf tmp/* +echo "removed content of tmp folder" + # stop the build if there are Python syntax errors or undefined names -if ! pipenv run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics; then +if ! flake8 --exclude venv --count --select=E9,F63,F7,F82 --show-source --statistics; then echo "There are flake8 problems [1]" popd > /dev/null exit 1 +else + echo "flake8-1 is done" fi # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide -if ! pipenv run flake8 . --count --exit-zero --max-complexity=15 --max-line-length=127 --statistics --per-file-ignores='yacg/model/*.py:E501 E303 W391'; then +if ! flake8 --exclude venv --count --exit-zero --max-complexity=15 --max-line-length=127 --statistics --per-file-ignores='yacg/model/*.py:E501 E303 W391'; then echo "There are flake8 problems [2]" popd > /dev/null exit 1 +else + echo "flake8-2 is done" fi if ! python -m unittest discover tests "test_*.py"; then From 70a9bf045c921d6f6a18220540e72cfbb796483c Mon Sep 17 00:00:00 2001 From: Eiko Thomas Date: Thu, 16 Nov 2023 18:10:29 +0100 Subject: [PATCH 4/4] fix tests --- .github/workflows/__tests.yml | 8 ++++---- version.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/__tests.yml b/.github/workflows/__tests.yml index 26db826d..8960f323 100644 --- a/.github/workflows/__tests.yml +++ b/.github/workflows/__tests.yml @@ -13,16 +13,16 @@ jobs: python-version: '3.10' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pipenv flake8 + [[ -d venv ]] || virtualenv venv + source venv/bin/activate pipenv --python 3.10 pipenv install - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --exclude venv --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=15 --max-line-length=127 --statistics --per-file-ignores='yacg/model/*.py:E501 E303 W391' + flake8 . --exclude venv --count --exit-zero --max-complexity=15 --max-line-length=127 --statistics --per-file-ignores='yacg/model/*.py:E501 E303 W391' - name: Run and stuff from command line run: | bin/runGithubActions.sh \ No newline at end of file diff --git a/version.txt b/version.txt index 6abaeb2f..024b066c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -6.2.0 +6.2.1