diff --git a/.github/rename_project.sh b/.github/rename_project.sh index a420de4..9c4f910 100755 --- a/.github/rename_project.sh +++ b/.github/rename_project.sh @@ -17,9 +17,9 @@ echo "Description: $description"; echo "Renaming project..." original_author="maycuatroi" -original_name="evo_flow" -original_urlname="evo-flow" -original_description="Awesome evo_flow created by maycuatroi" +original_name="evoflow" +original_urlname="evoflow" +original_description="Awesome evoflow created by maycuatroi" # for filename in $(find . -name "*.*") for filename in $(git ls-files) do @@ -30,7 +30,7 @@ do echo "Renamed $filename" done -mv evo_flow $name +mv evoflow $name # This command runs only once on GHA! rm -rf .github/template.yml diff --git a/.gitignore b/.gitignore index 2d0fadb..23ccb18 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,4 @@ dmypy.json # templates .github/templates/* +/.idea/ diff --git a/ABOUT_THIS_TEMPLATE.md b/ABOUT_THIS_TEMPLATE.md index b6dfd8b..58d3fc1 100644 --- a/ABOUT_THIS_TEMPLATE.md +++ b/ABOUT_THIS_TEMPLATE.md @@ -32,7 +32,7 @@ Lets take a look at the structure of this template: ├── Makefile # A collection of utilities to manage the project ├── MANIFEST.in # A list of files to include in a package ├── mkdocs.yml # Configuration for documentation site -├── evo_flow # The main python package for the project +├── evoflow # The main python package for the project │   ├── base.py # The base module for the project │   ├── __init__.py # This tells Python that this is a package │   ├── __main__.py # The entry point for the project @@ -109,7 +109,7 @@ I had to do some tricks to read that version variable inside the setuptools I decided to keep the version in a static file because it is easier to read from wherever I want without the need to install the package. -e.g: `cat evo_flow/VERSION` will get the project version without harming +e.g: `cat evoflow/VERSION` will get the project version without harming with module imports or anything else, it is useful for CI, logs and debugging. ### Why to include `tests`, `history` and `Containerfile` as part of the release? diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e60b1fa..3a75659 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # How to develop on this project -evo_flow welcomes contributions from the community. +evoflow welcomes contributions from the community. **You need PYTHON3!** diff --git a/Containerfile b/Containerfile index f435dbb..e212867 100644 --- a/Containerfile +++ b/Containerfile @@ -2,4 +2,4 @@ FROM python:3.7-alpine COPY . /app WORKDIR /app RUN pip install . -CMD ["evo_flow"] +CMD ["evoflow"] diff --git a/MANIFEST.in b/MANIFEST.in index 7c45962..398e2c4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,4 @@ include LICENSE include HISTORY.md include Containerfile graft tests -graft evo_flow +graft evoflow diff --git a/Makefile b/Makefile index 0dd9444..a8f0434 100644 --- a/Makefile +++ b/Makefile @@ -26,20 +26,20 @@ install: ## Install the project in dev mode. .PHONY: fmt fmt: ## Format code using black & isort. - $(ENV_PREFIX)isort evo_flow/ - $(ENV_PREFIX)black -l 79 evo_flow/ + $(ENV_PREFIX)isort evoflow/ + $(ENV_PREFIX)black -l 79 evoflow/ $(ENV_PREFIX)black -l 79 tests/ .PHONY: lint lint: ## Run pep8, black, mypy linters. - $(ENV_PREFIX)flake8 evo_flow/ - $(ENV_PREFIX)black -l 79 --check evo_flow/ + $(ENV_PREFIX)flake8 evoflow/ + $(ENV_PREFIX)black -l 79 --check evoflow/ $(ENV_PREFIX)black -l 79 --check tests/ - $(ENV_PREFIX)mypy --ignore-missing-imports evo_flow/ + $(ENV_PREFIX)mypy --ignore-missing-imports evoflow/ .PHONY: test test: lint ## Run tests and generate coverage report. - $(ENV_PREFIX)pytest -v --cov-config .coveragerc --cov=evo_flow -l --tb=short --maxfail=1 tests/ + $(ENV_PREFIX)pytest -v --cov-config .coveragerc --cov=evoflow -l --tb=short --maxfail=1 tests/ $(ENV_PREFIX)coverage xml $(ENV_PREFIX)coverage html @@ -78,9 +78,9 @@ virtualenv: ## Create a virtual environment. release: ## Create a new tag for release. @echo "WARNING: This operation will create s version tag and push to github" @read -p "Version? (provide the next x.y.z semver) : " TAG - @echo "$${TAG}" > evo_flow/VERSION + @echo "$${TAG}" > evoflow/VERSION @$(ENV_PREFIX)gitchangelog > HISTORY.md - @git add evo_flow/VERSION HISTORY.md + @git add evoflow/VERSION HISTORY.md @git commit -m "release: version $${TAG} 🚀" @echo "creating git tag : $${TAG}" @git tag $${TAG} @@ -101,7 +101,7 @@ switch-to-poetry: ## Switch to poetry package manager. @poetry init --no-interaction --name=a_flask_test --author=rochacbruno @echo "" >> pyproject.toml @echo "[tool.poetry.scripts]" >> pyproject.toml - @echo "evo_flow = 'evo_flow.__main__:main'" >> pyproject.toml + @echo "evoflow = 'evoflow.__main__:main'" >> pyproject.toml @cat requirements.txt | while read in; do poetry add --no-interaction "$${in}"; done @cat requirements-test.txt | while read in; do poetry add --no-interaction "$${in}" --dev; done @poetry install --no-interaction @@ -109,7 +109,7 @@ switch-to-poetry: ## Switch to poetry package manager. @mv requirements* .github/backup @mv setup.py .github/backup @echo "You have switched to https://python-poetry.org/ package manager." - @echo "Please run 'poetry shell' or 'poetry run evo_flow'" + @echo "Please run 'poetry shell' or 'poetry run evoflow'" .PHONY: init init: ## Initialize the project based on an application template. diff --git a/README.md b/README.md index 777299d..275b68a 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,28 @@ - -# Python Project Template - -A low dependency and really simple to start project template for Python Projects. - -See also -- [Flask-Project-Template](https://github.com/rochacbruno/flask-project-template/) for a full feature Flask project including database, API, admin interface, etc. -- [FastAPI-Project-Template](https://github.com/rochacbruno/fastapi-project-template/) The base to start an openapi project featuring: SQLModel, Typer, FastAPI, JWT Token Auth, Interactive Shell, Management Commands. - -### HOW TO USE THIS TEMPLATE - -> **DO NOT FORK** this is meant to be used from **[Use this template](https://github.com/rochacbruno/python-project-template/generate)** feature. - -1. Click on **[Use this template](https://github.com/rochacbruno/python-project-template/generate)** -3. Give a name to your project - (e.g. `my_awesome_project` recommendation is to use all lowercase and underscores separation for repo names.) -3. Wait until the first run of CI finishes - (Github Actions will process the template and commit to your new repo) -4. If you want [codecov](https://about.codecov.io/sign-up/) Reports and Automatic Release to [PyPI](https://pypi.org) - On the new repository `settings->secrets` add your `PYPI_API_TOKEN` and `CODECOV_TOKEN` (get the tokens on respective websites) -4. Read the file [CONTRIBUTING.md](CONTRIBUTING.md) -5. Then clone your new project and happy coding! - -> **NOTE**: **WAIT** until first CI run on github actions before cloning your new project. - -### What is included on this template? - -- 🖼️ Templates for starting multiple application types: - * **Basic low dependency** Python program (default) [use this template](https://github.com/rochacbruno/python-project-template/generate) - * **Flask** with database, admin interface, restapi and authentication [use this template](https://github.com/rochacbruno/flask-project-template/generate). - **or Run `make init` after cloning to generate a new project based on a template.** -- 📦 A basic [setup.py](setup.py) file to provide installation, packaging and distribution for your project. - Template uses setuptools because it's the de-facto standard for Python packages, you can run `make switch-to-poetry` later if you want. -- 🤖 A [Makefile](Makefile) with the most useful commands to install, test, lint, format and release your project. -- 📃 Documentation structure using [mkdocs](http://www.mkdocs.org) -- 💬 Auto generation of change log using **gitchangelog** to keep a HISTORY.md file automatically based on your commit history on every release. -- 🐋 A simple [Containerfile](Containerfile) to build a container image for your project. - `Containerfile` is a more open standard for building container images than Dockerfile, you can use buildah or docker with this file. -- 🧪 Testing structure using [pytest](https://docs.pytest.org/en/latest/) -- ✅ Code linting using [flake8](https://flake8.pycqa.org/en/latest/) -- 📊 Code coverage reports using [codecov](https://about.codecov.io/sign-up/) -- 🛳️ Automatic release to [PyPI](https://pypi.org) using [twine](https://twine.readthedocs.io/en/latest/) and github actions. -- 🎯 Entry points to execute your program using `python -m ` or `$ evo_flow` with basic CLI argument parsing. -- 🔄 Continuous integration using [Github Actions](.github/workflows/) with jobs to lint, test and release your project on Linux, Mac and Windows environments. - -> Curious about architectural decisions on this template? read [ABOUT_THIS_TEMPLATE.md](ABOUT_THIS_TEMPLATE.md) -> If you want to contribute to this template please open an [issue](https://github.com/rochacbruno/python-project-template/issues) or fork and send a PULL REQUEST. - -[❤️ Sponsor this project](https://github.com/sponsors/rochacbruno/) - - - ---- -# evo_flow +# evoflow [![codecov](https://codecov.io/gh/maycuatroi/evo-flow/branch/main/graph/badge.svg?token=evo-flow_token_here)](https://codecov.io/gh/maycuatroi/evo-flow) [![CI](https://github.com/maycuatroi/evo-flow/actions/workflows/main.yml/badge.svg)](https://github.com/maycuatroi/evo-flow/actions/workflows/main.yml) -Awesome evo_flow created by maycuatroi +a lite weight RPA and Workflow engine ## Install it from PyPI ```bash -pip install evo_flow +pip install evoflow ``` ## Usage ```py -from evo_flow import BaseClass -from evo_flow import base_function +from evoflow import BaseClass +from evoflow import base_function BaseClass().base_method() base_function() ``` ```bash -$ python -m evo_flow +$ python -m evoflow #or $ evo_flow ``` diff --git a/evo_flow/VERSION b/evo_flow/VERSION deleted file mode 100644 index 6e8bf73..0000000 --- a/evo_flow/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.0 diff --git a/evo_flow/__init__.py b/evoflow/Services/__init__.py similarity index 100% rename from evo_flow/__init__.py rename to evoflow/Services/__init__.py diff --git a/evoflow/Services/abstract_service.py b/evoflow/Services/abstract_service.py new file mode 100644 index 0000000..1e749f1 --- /dev/null +++ b/evoflow/Services/abstract_service.py @@ -0,0 +1,18 @@ +import abc + + +class AbstractService: + def __init__(self): + pass + + @abc.abstractmethod + def run(self, data, **args): + pass + + @abc.abstractmethod + def start(self, **args): + pass + + @abc.abstractmethod + def kill(self, **args): + pass diff --git a/evoflow/Services/ocr/__init__.py b/evoflow/Services/ocr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/Services/ocr/easy_ocr_engine.py b/evoflow/Services/ocr/easy_ocr_engine.py new file mode 100644 index 0000000..356b736 --- /dev/null +++ b/evoflow/Services/ocr/easy_ocr_engine.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +import logging +import os.path +from typing import Iterator +from urllib.error import URLError + +import numpy as np + +from evoflow import logger +from evoflow.Services.ocr.result import OCRResult + +try: + import easyocr +except ImportError: + logger.error("Can't import easyocr. Try to install with: pip install easyocr") + +MODEL_PATH = "/ocr" + + +class EasyOCREngine: + """ + Engine OCR sử dụng thư viện easyocr + """ + + def __init__(self, languages=None): + if languages is None: + languages = ["en"] + try: + model_storage_directory_evoflow = f'{os.getenv("userprofile")}/.evoflow/ocr' + model_storage_directory_data = "./data/.evoflow/ocr" + if os.path.isfile(model_storage_directory_evoflow): + model_storage_directory = model_storage_directory_evoflow + else: + model_storage_directory = model_storage_directory_data + self.reader = easyocr.Reader( + languages, + download_enabled=True, + model_storage_directory=model_storage_directory, + ) + except URLError as url_error: + logger.error("Can't download model, internet blocked") + raise URLError( + reason="Can't download model, internet blocked" + ) from url_error + logging.info("Initiated EasyOCREngine") + + def ocr(self, image: np.array, detail=0) -> Iterator[OCRResult]: + """ + + :param image: opencv image format + :param detail: + 0 for simpler output. + 1 for detail output. + """ + results = self.reader.readtext(np.array(image), detail=detail) + + for i, result in enumerate(results): + bbox = result[0] + bbox = [list(map(int, z)) for z in bbox] # convert np.int32 to int + res = (bbox, result[1], result[2]) + results[i] = OCRResult(*res) + return results + + @staticmethod + def supported_languages() -> Iterator[str]: + all_lang_list = easyocr.easyocr.all_lang_list + return all_lang_list diff --git a/evoflow/Services/ocr/ocr_service.py b/evoflow/Services/ocr/ocr_service.py new file mode 100644 index 0000000..23cecf3 --- /dev/null +++ b/evoflow/Services/ocr/ocr_service.py @@ -0,0 +1,56 @@ +from typing import Iterator + +import PIL +import numpy as np +from PIL import Image + +from evoflow.controller.log_controller import logger +from evoflow.entities.data_manipulate.file_operator.file import File +from evoflow.entities.data_manipulate.file_operator.image_file import ImageFile +from evoflow.Services.abstract_service import AbstractService +from evoflow.Services.ocr.Result import OCRResult + + +def get_image(data: File): + pil_image = np.array(Image.open(data.file)) + return pil_image + + +class OCRService(AbstractService): + def start(self, **args): + pass + + def kill(self, **args): + pass + + def run(self, data: File, **args): + + image = get_image(data) + result = self.ocr(image, **args) + return result + + def __init__(self, engine_name="easyocr", languages=None): + if languages is None: + languages = ["en", "ja"] + self.engine = None + if engine_name == "easyocr": + # pylint:disable=import-outside-toplevel + from evoflow.Services.OCR.EasyOCREngine import EasyOCREngine + + self.engine = EasyOCREngine(languages=languages) + else: + logger.error("OCR engine name not valid") + + super().__init__() + + def ocr(self, image_file: ImageFile, **args) -> Iterator[OCRResult]: + """ + OCR text from image. + :param image_file: Image file format + """ + if isinstance(image_file, str): + image = PIL.Image.open(image_file) + else: + image = image_file.to_pil() + results = self.engine.ocr(image, **args) + return results diff --git a/evoflow/Services/ocr/requirements.txt b/evoflow/Services/ocr/requirements.txt new file mode 100644 index 0000000..8b86553 --- /dev/null +++ b/evoflow/Services/ocr/requirements.txt @@ -0,0 +1 @@ +easyocr \ No newline at end of file diff --git a/evoflow/Services/ocr/result.py b/evoflow/Services/ocr/result.py new file mode 100644 index 0000000..320269e --- /dev/null +++ b/evoflow/Services/ocr/result.py @@ -0,0 +1,40 @@ +import numpy as np + + +class Bounding: + def __init__(self, bbox): + bbox = np.array(bbox) + self.bbox = bbox + self.xmin = min(bbox[:, 0]) + self.xmax = max(bbox[:, 0]) + self.ymin = min(bbox[:, 1]) + self.ymax = max(bbox[:, 1]) + + def get_center(self): + center = (self.xmax + self.xmin) / 2, (self.ymax + self.ymin) / 2 + return np.array(center) + + def compute_distance(self, bbox): + _c1 = self.get_center() + _c2 = bbox.get_center() + distance = (_c1 - _c2) ** 2 + return sum(distance) ** (1 / 2) + + def shift_x(self, pixcel): + self.bbox[:, 0] += pixcel + self.xmin += pixcel + self.xmax += pixcel + + def shift_y(self, pixcel): + pass + + +# pylint: disable=too-few-public-methods +class OCRResult: + def __init__(self, bbox, text, confidence): + self.bbox = Bounding(bbox) + self.text = text + self.confidence = confidence + + def __repr__(self): + return f"{self.text}, {self.confidence}" diff --git a/evoflow/Services/service_controller.py b/evoflow/Services/service_controller.py new file mode 100644 index 0000000..41c20ae --- /dev/null +++ b/evoflow/Services/service_controller.py @@ -0,0 +1,37 @@ +from evoflow.Services.ocr.ocr_service import OCRService + + +class ServicesController: + + def __init__(self): + self.running_services = {} + self.SERVICES_LIST = {'ocr': OCRService} + + def start_services(self, service_id, **args): + try: + service = self.SERVICES_LIST[service_id](**args) + self.running_services[service_id] = service + except Exception as e: + return e + return True + + def get_services_name(self): + services_names = list(self.SERVICES_LIST.keys()) + return services_names + + def run_service(self, service_id, data, **args): + if service_id not in self.running_services: + return 'Service not started' + sevice = self.running_services[service_id] + result = sevice.run(data, **args) + return result + + +if __name__ == '__main__': + import cv2 + + service = ServicesController().start_services('ocr', engine_name='easyocr', languages=['en', 'ja']) + service: OCRService + mat = cv2.imread('data/document_jp.PNG') + res = service.ocr(mat) + print(res) diff --git a/evoflow/Services/templates/item.html b/evoflow/Services/templates/item.html new file mode 100644 index 0000000..8a0d97a --- /dev/null +++ b/evoflow/Services/templates/item.html @@ -0,0 +1,9 @@ + + + Item Details + + + +

Item ID: {{ id }}

+ + \ No newline at end of file diff --git a/evoflow/Utils/__init__.py b/evoflow/Utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/Utils/create_session.py b/evoflow/Utils/create_session.py new file mode 100644 index 0000000..74f695f --- /dev/null +++ b/evoflow/Utils/create_session.py @@ -0,0 +1,5 @@ +from evoflow.entities.core.session import Session + + +def create_session(*args,**kwargs): + return Session(*args,**kwargs) \ No newline at end of file diff --git a/evoflow/Utils/ultilities.py b/evoflow/Utils/ultilities.py new file mode 100644 index 0000000..bb9646b --- /dev/null +++ b/evoflow/Utils/ultilities.py @@ -0,0 +1,37 @@ +import pathlib +import urllib + +import evoflow + +try: + from pyunpack import Archive +except ImportError: + evoflow.logger.error( + "Can't import pyunpack. Try to install with:\npip install pyunpack" + ) + + +def download_and_unzip(url: str, des_path: str, proxies_list: dict = None): + if proxies_list is None: + proxies_list = {} + pathlib.Path(des_path).mkdir(parents=True, exist_ok=True) + download_file_name = url.split("/")[-1] + opener = urllib.request.URLopener(proxies=proxies_list) + opener.addheader("User-Agent", "evoflow") + filename, _ = opener.retrieve(url, f"{des_path}/{download_file_name}") + Archive(filename).extractall(des_path) + os.remove(filename) + + +if __name__ == "__main__": + import os + + proxies = { + "http": "fsoft-proxy:8080", + } + downloaded_path = f'{os.getenv("userprofile")}/.evoflow' + download_and_unzip( + "https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip", + downloaded_path, + proxies_list=proxies, + ) diff --git a/evoflow/VERSION b/evoflow/VERSION new file mode 100644 index 0000000..f5d3d8c --- /dev/null +++ b/evoflow/VERSION @@ -0,0 +1 @@ +0.1.6.289 diff --git a/evoflow/__info__.py b/evoflow/__info__.py new file mode 100644 index 0000000..d98cfa0 --- /dev/null +++ b/evoflow/__info__.py @@ -0,0 +1,5 @@ +__author__ = "evoflow" +__maintainer__ = "Nguyen Anh binh" +__email__ = "sometimesocrazy@gmail.com" +__status__ = "developing" +__website__ = "https://github.com/maycuatroi/evo-flow/" \ No newline at end of file diff --git a/evoflow/__init__.py b/evoflow/__init__.py new file mode 100644 index 0000000..4c1f1bb --- /dev/null +++ b/evoflow/__init__.py @@ -0,0 +1,69 @@ +import inspect +import os + +os.environ['GIT_PYTHON_REFRESH'] = 'quiet' +import sys + +from evoflow import params +from evoflow import version as evoflow_version +from evoflow.controller.log_controller import logger + +import git +from evoflow.entities.core.job import Job +from evoflow.entities.core.step import Step +from evoflow.__info__ import __website__ + +logger.debug(f'Start evoflow v {evoflow_version.__version__}. You can visit {__website__} for more information') + +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, parentdir) + + +def init_setting(): + try: + import setting + for key in setting.__dict__: + if str(key).startswith('__'): + continue + value = setting.__dict__[key] + os.environ[key] = str(value) + except Exception as e: + logger.debug(e) + logger.info("Can't import setting.py") + + +def versioning(version_module): + try: + repo = git.Repo(search_parent_directories=True) + sha = repo.head.object.hexsha + if version_module.__sha__ != sha: + commits = list(repo.iter_commits('HEAD')) + total_commit = len(commits) + version_parts = version_module.__version__.split('.') + version_parts[3] = str(total_commit) + version_path = inspect.getfile(version_module) + version_lines = [f"__version__ = '{'.'.join(version_parts)}'\n", + f"__sha__ = '{sha}'"] + open(version_path, 'w').writelines(version_lines) + except: + logger.error("Can't versioning") + + +def framework_versioning(): + versioning(evoflow_version) + + +def bot_versioning(): + try: + import version + versioning(version) + except: + logger.error("Can't versioning For Bot") + + +init_setting() +# framework_versioning() +bot_versioning() + +IS_DEBUG = sys.gettrace() is not None diff --git a/evo_flow/__main__.py b/evoflow/__main__.py similarity index 76% rename from evo_flow/__main__.py rename to evoflow/__main__.py index dd25939..1cf7bf5 100644 --- a/evo_flow/__main__.py +++ b/evoflow/__main__.py @@ -1,4 +1,4 @@ -"""Entry point for evo_flow.""" +"""Entry point for evoflow.""" from .cli import main # pragma: no cover diff --git a/evo_flow/base.py b/evoflow/base.py similarity index 71% rename from evo_flow/base.py rename to evoflow/base.py index 12553ee..a7b1817 100644 --- a/evo_flow/base.py +++ b/evoflow/base.py @@ -1,7 +1,7 @@ """ -evo_flow base module. + base module. -This is the principal module of the evo_flow project. +This is the principal module of the evoflow project. here you put your main classes and objects. Be creative! do whatever you want! @@ -14,4 +14,4 @@ """ # example constant variable -NAME = "evo_flow" +NAME = "evoflow" diff --git a/evo_flow/cli.py b/evoflow/cli.py similarity index 88% rename from evo_flow/cli.py rename to evoflow/cli.py index 8732765..976ce2f 100644 --- a/evo_flow/cli.py +++ b/evoflow/cli.py @@ -1,4 +1,4 @@ -"""CLI interface for evo_flow project. +"""CLI interface for evoflow project. Be creative! do whatever you want! @@ -12,7 +12,7 @@ def main(): # pragma: no cover """ The main function executes on commands: - `python -m evo_flow` and `$ evo_flow `. + `python -m evoflow` and `$ evoflow `. This is your program's entry point. diff --git a/evoflow/controller/__init__.py b/evoflow/controller/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/controller/common_step_controller.py b/evoflow/controller/common_step_controller.py new file mode 100644 index 0000000..27f9360 --- /dev/null +++ b/evoflow/controller/common_step_controller.py @@ -0,0 +1,15 @@ +from evoflow.Entities.Common import Step + + +def get_all_common_steps(): + return Step.get_all_common_steps() + + +def get_all_common_steps_name(): + return [step.name for step in get_all_common_steps()] + + +if __name__ == '__main__': + data = get_all_common_steps() + for key, value in data.items(): + print(f'{key} - {str(value)}') diff --git a/evoflow/controller/data_manipulate/PdfFileOperator.py b/evoflow/controller/data_manipulate/PdfFileOperator.py new file mode 100644 index 0000000..9235e32 --- /dev/null +++ b/evoflow/controller/data_manipulate/PdfFileOperator.py @@ -0,0 +1,23 @@ +import pdfplumber + +from evoflow.Controller.DataManipulate.FileOperator import FileOperator +from evoflow.Entities.DataManipulate.FileOperator.File import File +from evoflow.Entities.DataManipulate.FileOperator.PdfFile import PdfFile + + +class PdfFileOperator(FileOperator): + """ + Read and write file PDF + """ + + def save(self, file: File, file_path) -> bool: + pass + + def read(self, file_path, **args): + file_data = pdfplumber.open(file_path) + file = PdfFile(data=file_data, file_path=file_path) + return file + + @staticmethod + def supported_file_types(): + return ['pdf'] diff --git a/evoflow/controller/data_manipulate/PptxFileOperator.py b/evoflow/controller/data_manipulate/PptxFileOperator.py new file mode 100644 index 0000000..fa23aab --- /dev/null +++ b/evoflow/controller/data_manipulate/PptxFileOperator.py @@ -0,0 +1,21 @@ +from pptx import Presentation + +from evoflow.Controller.DataManipulate.FileOperator import FileOperator +from evoflow.Entities.DataManipulate.FileOperator.File import File +from evoflow.Entities.DataManipulate.FileOperator.PPTXFile import PPTXFile + + +class PptxFileOperator(FileOperator): + def save(self, file: File, file_path) -> bool: + file.data: Presentation + file.data.save(file_path) + return True + + def read(self, file_path, **args): + file = PPTXFile(file_path=file_path) + file.data = Presentation(file_path) + return file + + @staticmethod + def supported_file_types(): + return ['ppt', 'pptx'] diff --git a/evoflow/controller/data_manipulate/__init__.py b/evoflow/controller/data_manipulate/__init__.py new file mode 100644 index 0000000..6e7245c --- /dev/null +++ b/evoflow/controller/data_manipulate/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +from evoflow.Controller.DataManipulate.ExcelFileOperator import ExcelFileOperator +from evoflow.Controller.DataManipulate.PdfFileOperator import PdfFileOperator +from evoflow.Controller.DataManipulate.ImageFileOperator import ImageFileOperator +from evoflow.Controller.DataManipulate.PptxFileOperator import PptxFileOperator +from evoflow.Controller.DataManipulate.CatiaFileOperator import CatiaFileOperator diff --git a/evoflow/controller/data_manipulate/data_manipulate.py b/evoflow/controller/data_manipulate/data_manipulate.py new file mode 100644 index 0000000..9ee5716 --- /dev/null +++ b/evoflow/controller/data_manipulate/data_manipulate.py @@ -0,0 +1,20 @@ +from abc import abstractmethod + + +class DataManipulate: + """ + Class object xử lý DATA + """ + + def authenticate(self, **args): + """ + Thực hiện authenticate với data source + :param args: + :return: + """ + + @abstractmethod + def read(self, file_path: str, **args): + """ + Read file + """ diff --git a/evoflow/controller/data_manipulate/excel_file_operator.py b/evoflow/controller/data_manipulate/excel_file_operator.py new file mode 100644 index 0000000..8f5307d --- /dev/null +++ b/evoflow/controller/data_manipulate/excel_file_operator.py @@ -0,0 +1,36 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +import pandas as pd + +from evoflow.controller.data_manipulate.FileOperator import FileOperator +from evoflow.Entities.DataManipulate.FileOperator.DataFrameFile import DataFrameFile +from evoflow.Entities.DataManipulate.FileOperator.File import File + + +class ExcelFileOperator(FileOperator): + + def read(self, file_path, **args) -> File: + """ + read file excel + :param file_path: + :return: + """ + + file = DataFrameFile(file_path=file_path) + file.data = pd.read_excel(file_path, **args) + return file + + def save(self, file: File, file_path) -> bool: + """ + save excel file + :param file: + :param file_path: + :return: + """ + file.data: pd.DataFrame + file.data.to_excel(file_path) + return file_path + + @staticmethod + def supported_file_types(): + return ['xlsx', 'csv', 'xls'] diff --git a/evoflow/controller/data_manipulate/file_operator.py b/evoflow/controller/data_manipulate/file_operator.py new file mode 100644 index 0000000..973171f --- /dev/null +++ b/evoflow/controller/data_manipulate/file_operator.py @@ -0,0 +1,95 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +import abc + +from tqdm import tqdm + +from evoflow.Controller.DataManipulate.DataManipulate import DataManipulate +from evoflow.Entities.DataManipulate.FileOperator.File import File + + +def get_reader(file_type): + """ + Trả về reader thỏa vãn mới file_type này + """ + readers = FileOperator.__subclasses__() + for reader in readers: + if file_type in reader.supported_file_types(): + return reader() + print(f"Can't read file with file type: .{file_type}") + return None + + +def get_file_type(file_path): + """ + Trả về file type của file_path + """ + file_type = file_path.split('.')[-1].lower() + return file_type + + +class FileOperator(DataManipulate): + """ + Operator để thao tác xử lý files + """ + + def read(self, file_path, **args) -> File: + """ + đọc file, tự lựa chọn cách đọc phù hợp cho các loại files + :param file_path: đường dẫn tới file + :args: Params bổ sung + :return: Return None nếu đọc không thành công + """ + + file_type = get_file_type(file_path) + reader = get_reader(file_type) + return reader.read(file_path, **args) + + def reads(self, file_paths) -> list: + """ + Đọc nhiều files khác nhau theo đường dẫn + :param file_paths: list file paths + :return: + """ + results = [] + for file_path in tqdm(file_paths, desc="Read files"): + file_data = self.read(file_path) + if file_data is not None: + results.append(file_data) + return results + + @abc.abstractmethod + def save(self, file: File, file_path) -> bool: + """ + Luư file + :param file_path: Đường dẫn tới file + :return: + """ + + def copy(self, file_source, file_destination, overwrite=False) -> str: + """ + Coppy file from file_source to file_destination. + :param overwrite: True thì ghi đè + :param file_source: đường dẫn tới source file. + :param file_destination: đường dẫn nơi copy tới + :return: Đường dẫn tới file sau khi Coppy + """ + + def remove(self, file_path, safe_delete=False) -> bool: + """ + Xóa file tại đường dẫn file_path + Nếu file_path là đường dẫn của folder thì xóa folder, nếu là file thì xóa file. + + :param file_path: đường dẫn tới file + :param safe_delete: không xóa khi file đang được mở + :return: True nếu xóa thành công + """ + + @staticmethod + @abc.abstractmethod + def supported_file_types(): + """ + trả về các loại file mà reader này hỗ trợ + vd: ['xlsx','csv'] + """ + return [] diff --git a/evoflow/controller/data_manipulate/image_file_operator.py b/evoflow/controller/data_manipulate/image_file_operator.py new file mode 100644 index 0000000..d726bf0 --- /dev/null +++ b/evoflow/controller/data_manipulate/image_file_operator.py @@ -0,0 +1,52 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +import os + +import cv2 +import numpy as np + +from evoflow import logger +from evoflow.controller.data_manipulate.file_operator import FileOperator +from evoflow.entities.data_manipulate.file_operator.file import File +from evoflow.entities.data_manipulate.file_operator.ImageFile import ImageFile + + +def imread(path, flags: int = -1): + """ + :flags: Default= cv2.IMREAD_UNCHANGED + """ + normed_path = os.path.normpath(path) + mat = cv2.imdecode(np.fromfile(normed_path, dtype=np.uint8), flags) + if mat is None: + logger.error(f"Can't read image: {path}") + raise Exception + return mat + + +def imwrite(path, mat): + image_type = path.split('.')[-1] + is_success, im_buf_arr = cv2.imencode(".{}".format(image_type), mat) + im_buf_arr.tofile(path) + return is_success + + +cv2.imwrite = imwrite +cv2.imread = imread + + +class ImageFileOperator(FileOperator): + """ + File operator with images + """ + + def save(self, file: File, file_path) -> bool: + return cv2.imwrite(file_path, file.data) + + def read(self, file_path, **args): + data = cv2.imread(file_path, **args) + file = ImageFile(file_path=file_path, data=data) + return file + + @staticmethod + def supported_file_types(): + return ['jpg', 'png', 'tiff', 'tif'] diff --git a/evoflow/controller/log_controller.py b/evoflow/controller/log_controller.py new file mode 100644 index 0000000..2709318 --- /dev/null +++ b/evoflow/controller/log_controller.py @@ -0,0 +1,47 @@ +import logging +import pathlib +import platform +import sys +import time + +import coloredlogs + +pathlib.Path('logs').mkdir(parents=True, exist_ok=True) +logger = logging.getLogger() +current_time_string = time.strftime('%Y%m%d-%H%M%S') +FILE_NAME = 'logs/evoflow.log' +fh = logging.FileHandler(filename=FILE_NAME, mode='a', encoding='utf-8') +fh.setLevel(logging.ERROR) +formatter = coloredlogs.ColoredFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +logger.addHandler(fh) +coloredlogs.install(level=logging.INFO, logger=logger) + + +def pretty_dict(dict_object, indent=0): + pretty_string = '\n' + for key, value in dict_object.items(): + pretty_string += '\t' * indent + str(key) + ' : ' + pretty_string += '\t' * (indent + 1) + str(value) + '\n' + return pretty_string + + +def get_os_information(): + os_information = """Python version: %s + dist: %s + system: %s + machine: %s + platform: %s + uname: %s + version: %s + """ % ( + sys.version.split('\n'), + str(platform.win32_edition()), + platform.system(), + platform.machine(), + platform.platform(), + platform.uname(), + platform.version(), + ) + + return os_information diff --git a/evoflow/controller/visualize_controller.py b/evoflow/controller/visualize_controller.py new file mode 100644 index 0000000..fe845eb --- /dev/null +++ b/evoflow/controller/visualize_controller.py @@ -0,0 +1,13 @@ +from PIL import ImageDraw +from PIL import ImageFont + + +def draw_boxes(image, bounds, color='yellow', width=2): + draw = ImageDraw.Draw(image) + font = ImageFont.truetype("data/HachiMaruPop-Regular.ttf", 32) + for bound in bounds: + p_0, p_1, p_2, p_3 = bound[0] + text = bound[1] + draw.line([*p_0, *p_1, *p_2, *p_3, *p_0], fill=color, width=width) + draw.text(p_0, text, (255, 0, 0), font=font) + return image diff --git a/evoflow/entities/__init__.py b/evoflow/entities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/common/Function/__init__.py b/evoflow/entities/common/Function/__init__.py new file mode 100644 index 0000000..123d94c --- /dev/null +++ b/evoflow/entities/common/Function/__init__.py @@ -0,0 +1,27 @@ +from time import sleep + +from evoflow.Entities.Global import Global + + +def add_text_with_leader(i_object=None, text=''): + caa = Global().caa + selection = caa.active_document.selection + if i_object is not None: + selection.clear() + selection.add(i_object) + caa.start_command('Text with leader') + text_editor_window = caa.gui.window(title='Text Editor') + text_editor_window.Edit.set_edit_text(text) + text_editor_window.OK.click() + + +def disassemble(i_object=None): + caa = Global().caa + selection = caa.active_document.selection + if i_object is not None: + selection.clear() + selection.add(i_object) + caa.start_command('disassemble') + text_editor_window = Global().caa.gui.window(title='Disassemble') + text_editor_window.OK.click() + sleep(2) diff --git a/evoflow/entities/common/Step/Brower/__init__.py b/evoflow/entities/common/Step/Brower/__init__.py new file mode 100644 index 0000000..8927654 --- /dev/null +++ b/evoflow/entities/common/Step/Brower/__init__.py @@ -0,0 +1,39 @@ +from selenium import webdriver +from selenium.webdriver import Proxy +from selenium.webdriver.common.proxy import ProxyType + +import evoflow + + +@evoflow.Step() +def open_browser(url: str = None, proxy: Proxy = None) -> dict: + """ + Open Fire fox browser + + :rtype: dict + :param url: + :param proxy: + :return: dict object + """ + browser = webdriver.Firefox(proxy=proxy) + browser.get(url) + return {'browser': browser} + + +@evoflow.Step() +def setup_proxy(proxy_url: str = None): + """ + + :param proxy_url: "xx.xx.xx.xx:xxxx" + :return: + """ + proxy = None + if proxy_url: + proxy = Proxy({ + 'proxyType': ProxyType.MANUAL, + 'httpProxy': proxy_url, + 'ftpProxy': proxy_url, + 'sslProxy': proxy_url, + 'noProxy': '' # set this value as desired + }) + return {'proxy': proxy} diff --git a/evoflow/entities/common/Step/CATIA/GetObjectsByName.py b/evoflow/entities/common/Step/CATIA/GetObjectsByName.py new file mode 100644 index 0000000..d781028 --- /dev/null +++ b/evoflow/entities/common/Step/CATIA/GetObjectsByName.py @@ -0,0 +1,9 @@ +from evoflow.Entities.Core.Step import Step +from evoflow.Entities.Global import Global + + +class GetObjectsByName(Step): + def action(self): + caa = Global().caa + caa.active_document.selection.search(f'Name={self.object_name},all') + return {'objects': caa.active_document.selection.items()} diff --git a/evoflow/entities/common/Step/CATIA/__init__.py b/evoflow/entities/common/Step/CATIA/__init__.py new file mode 100644 index 0000000..65f1f8a --- /dev/null +++ b/evoflow/entities/common/Step/CATIA/__init__.py @@ -0,0 +1,6 @@ +import evoflow +from evoflow.Controller.DataManipulate.FileOperator import FileOperator + +@evoflow.Step() +def open_catia_file(path: str = None): + FileOperator().read(path) diff --git a/evoflow/entities/common/Step/CATIA/export.py b/evoflow/entities/common/Step/CATIA/export.py new file mode 100644 index 0000000..276112e --- /dev/null +++ b/evoflow/entities/common/Step/CATIA/export.py @@ -0,0 +1,181 @@ +import os +import pathlib + +import pandas as pd +from tqdm import tqdm +from vedo import Spline + +import evoflow +from evoflow import Job, logger +from evoflow.Entities.Global import Global +from evoflow.pycatia.hybrid_shape_interfaces.hybrid_shape_curve_explicit import HybridShapeCurveExplicit +from evoflow.pycatia.mec_mod_interfaces.hybrid_shape import HybridShape + + +@evoflow.Step() +def hide_all(): + caa = Global().caa + selection = caa.active_document.selection + selection.clear() + selection.search("type=*,all") + selection.hide() + selection.clear() + + +@evoflow.Step() +def export_splines(spline_names: list = None, resolution: int = None): + """ + Export Splines ( Curve ) trong file CATIA đang mở + :param resolution: độ phân giải của gieo điểm để export splines Ex: 1 + :param spline_names: Tên hiển thị trên cây thư mục. Ex: Curve.319 dùng để export + :return: + """ + + caa = Global().caa + document = caa.active_document + selection = caa.active_document.selection + selection.clear() + hybrid_shape_factory = document.part().hybrid_shape_factory + if spline_names is not None: + z = [f"Name='{spline_name}'" for spline_name in spline_names] + search_query = ' + '.join(z) + search_query = f'({search_query}),all' + else: + # Search all curve + search_query = "((((((FreeStyle.Curve + '2D Layout for 3D Design'.Curve) + Sketcher.Curve) + Drafting.Curve) + 'Part Design'.Curve) + 'Generative Shape Design'.Curve) + 'Functional Molded Part'.Curve),all" + selection.search(search_query) + selected_items = selection.items() + logger.info(f"Found {len(selected_items)} Splines") + spa_workbench = document.spa_workbench() + + spline_points = {} + + evoflow_part = caa.active_document.create_hybrid('evoflow') + + for spline in selected_items: + spline = HybridShapeCurveExplicit(spline.com_object) + spline_points[spline.get_path()] = [] + resolution = resolution or os.getenv('CURVE_RESOLUTION') or input( + 'Enter curve resolution to export (mm)- Default = 10:') or 10 + resolution = int(resolution) + + points = spline.add_points_on_curve(distance=resolution, hybrid_shape_factory=hybrid_shape_factory) + + evoflow_part.append_hybrid_shapes(points) + evoflow_part.update() + spline.show(show_all_parent=True) + for point_on_curve in tqdm(points, desc=f'Export Curve {spline.name}'): + mesureable = point_on_curve.get_measurable(workbench=spa_workbench) + x, y, z = mesureable.get_point() + spline_points[spline.get_path()].append([x, y, z]) + spline.hide() + + splines = [] + for name, points in spline_points.items(): + try: + spline = Spline(points) + except: + logger.info(f"Can't export spline: {name}, try to add more points") + continue + splines.append(spline) + output_name = f"{os.getenv('DATA_PATH')}/{name}.vtk" + output_dir = os.path.split(output_name)[0] + pathlib.Path(output_dir).mkdir(exist_ok=True, parents=True) + spline.write(output_name) + evoflow_part.delete() + caa.start_command('clear history') + return {'splines': splines} + + +@evoflow.Step() +def export_surface_to_stl(): + mesh_path = os.getenv('DATA_PATH') or input('Enter mesh path: ') or '.' + + caa = Global().caa + document_name = '.'.join(caa.active_document.name.split('.')[:-1]) + mesh_path = os.path.abspath(mesh_path) + show_only_part_name = os.getenv('EXPORT_PART_NAME') or input('Enter RIB part name: ') + selection = caa.active_document.selection + selection.search(f"Name='{show_only_part_name}',all") + visible_items = selection.items() + wb = caa.active_document.spa_workbench() + for item in visible_items: + item.show(show_all_parent=True) + + selection.search(f"Name={show_only_part_name},all") + selection.search("type=*,sel") + rib_items = selection.items() + progress_bar = tqdm(rib_items, desc='Export RIB to STL') + for rib_item in progress_bar: + try: + shape = HybridShape(rib_item.com_object) + shape.compute() + except AttributeError as abe: + continue + + progress_bar.set_description(f'Export {rib_item.name} to STL') + measurable = shape.get_measurable(workbench=wb) + try: + area = measurable.area + except: + continue + shape.show(show_all_parent=True) + path = shape.get_path() + file_name = f"{mesh_path}/{path}.stl" + file_name = os.path.abspath(file_name) + caa.active_document.export_data(file_name, file_type='stl', overwrite=True) + rib_item.hide() + logger.info(f'Found {selection.count} items in {show_only_part_name}') + + return {'mesh_path': mesh_path} + + +@evoflow.Step() +def catia_to_evoflow(): + start_step = hide_all() + x = start_step.next(export_surface_to_stl()) + x = x.next(export_splines()) + + job = Job(name='Export All data from CATIA', start_step=start_step) + job.run() + return job.params_pool + + +@evoflow.Step() +def get_die_direction(): + caa = Global().caa + document = caa.active_document + selection = document.selection + selection.clear() + evoflow_exported_path = os.getenv('DATA_PATH') or input('Enter exported path: ') + die_direction_name = os.getenv('DIE_DIRECTION_NAME') or input('Enter Die direction name: ') + selection.search(f"Name={die_direction_name},all") + items = selection.items() + x, y, z = None, None, None + for die_vector in items: + try: + die_vector = HybridShape(die_vector.com_object) + logger.info(f'Found die direction items: {die_vector.get_path()}') + x, y, z = die_vector.get_measurable().get_direction() + px, py, pz = die_vector.get_measurable().get_points_on_curve()[:3] + except Exception as e: + logger.error(e) + + logger.info(f"get diecrion: {x, y, z}") + if x is None: + raise Exception("Can't find die direction object") + + df = pd.DataFrame( + data={'die_angle_x': [x], 'die_angle_y': [y], 'die_angle_z': [z], 'px': [px], 'py': [py], 'pz': [pz]}) + + pathlib.Path(evoflow_exported_path).mkdir(exist_ok=True) + df.to_excel(f'{evoflow_exported_path}/dieangle.xlsx', index=False) + + return {'die_angle_x': x, 'die_angle_y': y, 'die_angle_z': z, 'px': px, 'py': py, 'pz': pz} + + +if __name__ == '__main__': + job = Job( + start_step=export_splines(spline_names=['Curve.282']) + ) + job.run() diff --git a/evoflow/entities/common/Step/CNavi/ReadInputFile.py b/evoflow/entities/common/Step/CNavi/ReadInputFile.py new file mode 100644 index 0000000..8e1e73e --- /dev/null +++ b/evoflow/entities/common/Step/CNavi/ReadInputFile.py @@ -0,0 +1,10 @@ +from evoflow.Entities.Common.Step.OpenExcelFile import OpenExcelFile + + +class ReadInputFile(OpenExcelFile): + def __init__(self, **kwargs): + self.file_path = None + super().__init__(self.file_path, **kwargs) + + def prepare(self, config=None, **kwargs): + self.file_path = config['data_source'] + config['InputFile'] diff --git a/evoflow/entities/common/Step/CNavi/__init__.py b/evoflow/entities/common/Step/CNavi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/common/Step/Downloader.py b/evoflow/entities/common/Step/Downloader.py new file mode 100644 index 0000000..74ae54e --- /dev/null +++ b/evoflow/entities/common/Step/Downloader.py @@ -0,0 +1,92 @@ +import os +import pathlib +import shutil +from urllib.request import urlretrieve + +import pandas as pd +import requests +from tqdm import tqdm + +import evoflow +from evoflow import logger + + +class TqdmDownload(tqdm): + """Alternative Class-based version of the above. + Provides `update_to(n)` which uses `tqdm.update(delta_n)`. + Inspired by [twine#242](https://github.com/pypa/twine/pull/242), + [here](https://github.com/pypa/twine/commit/42e55e06). + """ + + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) # will also set self.n = b * bsize + + +@evoflow.Step() +def download_urls(urls: list = None, download_dir: str = 'Downloads') -> object: + """ + @param urls: List of urls to download + @param download_dir: Download directory + @return: download_paths: List path to downloaded files + @rtype: dict + + """ + download_paths = [] + pathlib.Path(download_dir).mkdir(parents=True, exist_ok=True) + progres_bar = tqdm(urls) + + for url in progres_bar: + progres_bar.set_description(f'Downloading: {url}') + try: + res = requests.get(url) + filename = 'zasdvadf' + with TqdmDownload(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, desc=filename) as t: + download_path, result = urlretrieve(url, f'{download_dir}/miniconda.exe') + download_paths.append(download_path) + + except: + logger.error(f"Can't access : {url}") + + return {'download_paths': download_path} + + +@evoflow.Step() +def download_urls(urls: list = None, download_dir: str = None) -> object: + """ + @param urls: List of urls to download + @param download_dir: Download directory + @return: download_paths: List path to downloaded files + @rtype: dict + + """ + download_paths = [] + df = pd.DataFrame(columns=['URL', 'FILE_NAME']) + if download_dir: + pathlib.Path(download_dir).mkdir(parents=True, exist_ok=True) + for i, url in tqdm(enumerate(urls), total=len(urls)): + try: + res = requests.get(url) + with TqdmDownload(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, desc=url) as t: + download_path, result = urlretrieve(url) + file_name = os.path.split(download_path)[1] + if download_dir: + shutil.move(download_path, f'{download_dir}/{file_name}') + download_path = f'{download_dir}/{file_name}' + download_paths.append(download_path) + + df.loc[i] = [url, download_path] + df.to_csv(f'{download_dir}/downloader_log.csv', encoding='utf-8', index=False) + except Exception as e: + logger.error(f"{e}") + + return {'download_files': download_path} diff --git a/evoflow/entities/common/Step/FileOperator/OpenCATIAFile.py b/evoflow/entities/common/Step/FileOperator/OpenCATIAFile.py new file mode 100644 index 0000000..9077c03 --- /dev/null +++ b/evoflow/entities/common/Step/FileOperator/OpenCATIAFile.py @@ -0,0 +1,29 @@ +import os + +from evoflow.Controller.DataManipulate.FileOperator import FileOperator +from evoflow.Entities.Core.Step import Step +from evoflow.Entities.Global import Global + + +class OpenCATIAFile(Step): + def __init__( + self, name: str = None, transactions=None, file_name: str = '', **kwargs + ): + ''' + + @param file_name: Đường dẫn tới CATIA File + ''' + super().__init__(name, transactions, **kwargs) + self.file_name = file_name + + def action(self, **kwargs): + if not self.params.get('FILE_NAME'): + print('[OPENING CATIA]') + self.file_name = os.getenv('CATIA_FILE_NAME') or input('Enter CATIA file name: ') + if self.file_name is None: + caa = Global().caa + return {'catia_file': caa.active_document} + self.file_name = self.file_name.strip('"') + file = FileOperator().read(self.file_name) + + return {'catia_file': file} diff --git a/evoflow/entities/common/Step/FileOperator/OpenPointPointApplication.py b/evoflow/entities/common/Step/FileOperator/OpenPointPointApplication.py new file mode 100644 index 0000000..31a250d --- /dev/null +++ b/evoflow/entities/common/Step/FileOperator/OpenPointPointApplication.py @@ -0,0 +1,19 @@ +import subprocess + +from evoflow.Entities.Core.Step import Step + + +class OpenPointPointApplication(Step): + """ + Open PowerPoint File + + """ + + def __init__(self, **kwargs): + """ + + @param file_path: path of powerpoint file + """ + + def action(self): + subprocess.Popen(r'"C:\Program Files\Microsoft Office\Office16\POWERPNT.EXE" ' + f'"{self.file_path}"') diff --git a/evoflow/entities/common/Step/FileOperator/__init__.py b/evoflow/entities/common/Step/FileOperator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/common/Step/Mesh/__init__.py b/evoflow/entities/common/Step/Mesh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/common/Step/OpenExcelFile.py b/evoflow/entities/common/Step/OpenExcelFile.py new file mode 100644 index 0000000..6df355f --- /dev/null +++ b/evoflow/entities/common/Step/OpenExcelFile.py @@ -0,0 +1,38 @@ +from evoflow.Controller.DataManipulate.FileOperator import FileOperator +from evoflow.Entities.Core.Step import Step + + +class OpenExcelFile(Step): + + def __init__(self, file_path='', **kwargs): + """ + @param file_path: + @param kwargs: pandas params + """ + super().__init__(name='Open excel file', **kwargs) + self.file_path = file_path + self.kwars = kwargs + + def end(self, **kwargs) -> dict: + return super().end(**kwargs) + + def prepare(self, **kwargs): + super().prepare(**kwargs) + + def summary(self, **kwargs): + return super().summary(**kwargs) + + def __info__(self, **kwargs) -> dict: + return super().__info__(**kwargs) + + def action(self, **kwargs): + reader = FileOperator() + file = reader.read(file_path=self.file_path, **self.kwars) + return {'file': file} + + +if __name__ == '__main__': + step_open_excel = OpenExcelFile(file_path='data/Input.xlsx', header=12) + results = step_open_excel.action() + summary = results['file'].summary() + print(summary) diff --git a/evoflow/entities/common/Step/__init__.py b/evoflow/entities/common/Step/__init__.py new file mode 100644 index 0000000..8d952ba --- /dev/null +++ b/evoflow/entities/common/Step/__init__.py @@ -0,0 +1,34 @@ +import glob +import inspect +import os + +import evoflow + + +def get_all_common_steps(): + current_dir = os.path.split(__file__)[0] + cwd = os.path.split(os.getcwd())[0] + file_paths = glob.glob(f'{current_dir}/*.py') + data = {} + for file_path in file_paths: + file_path = os.path.normpath(file_path) + file_path = file_path.replace(cwd, '') + if file_path.startswith('__init__'): + continue + + import_string = file_path.replace(os.sep, '.')[:-3][1:] + imported_module = __import__(import_string, fromlist=import_string.split('.')[-1]) + + for class_name, import_class in inspect.getmembers(imported_module): + if class_name == import_string.split('.')[-1]: + data[class_name] = import_class() + break + return data + + +@evoflow.Step() +def show_menu(menu_dict: dict = {}): + for key, value in menu_dict.items(): + print(f'{key}. {value}') + select_option = os.environ.get('SELECT_OPTION') or input('Enter your option: ') + return {'select_option': int(select_option)} diff --git a/evoflow/entities/common/__init__.py b/evoflow/entities/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/core/__init__.py b/evoflow/entities/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/core/base_object.py b/evoflow/entities/core/base_object.py new file mode 100644 index 0000000..ed25c9e --- /dev/null +++ b/evoflow/entities/core/base_object.py @@ -0,0 +1,47 @@ +# Copyright (c) 2021. Copyright belongs to evoflow team + +import uuid +from json import JSONEncoder + +import json_tricks as json + + +class BaseObject: + """ + Base object of all things + """ + + def __init__(self, name=None, **kwargs): + self.name = f'{self.__class__.__name__} {name}' + self.id = uuid.uuid4().hex + for key, value in kwargs.items(): + setattr(self, key, value) + + def summary(self, **kwargs): + info = self.__info__(**kwargs) + info['name'] = self.name + json_string = json.dumps(info, ensure_ascii=False, indent=2) + return json_string + + def kill(self, **kwargs): + pass + + def __info__(self) -> dict: + """ + + @rtype: return object info to display or log + @param kwargs: + """ + return {'name': self.name} + + def to_json(self): + data = self.__dict__ + json_string = json.dumps(data, ensure_ascii=False, indent=2) + return json_string + + class Encoder(JSONEncoder): + def default(self, o): + return o.to_dict() + + def to_dict(self): + return self.__dict__ diff --git a/evoflow/entities/core/condition.py b/evoflow/entities/core/condition.py new file mode 100644 index 0000000..54121f6 --- /dev/null +++ b/evoflow/entities/core/condition.py @@ -0,0 +1,19 @@ +import inspect + +from evoflow.entities.core.base_object import BaseObject + + +class Condition(BaseObject): + def __init__(self, condition_function): + """ + + @param condition_function: Boolean Validate function + """ + self.condition_function = condition_function + + def to_dict(self): + if type(self.condition_function) != str: + args = inspect.getfullargspec(self.condition_function).args + z = inspect.getsource(self.condition_function) + return {'condition_function': {'args': args, 'source': z}} + return self.__dict__ diff --git a/evoflow/entities/core/job.py b/evoflow/entities/core/job.py new file mode 100644 index 0000000..c361dae --- /dev/null +++ b/evoflow/entities/core/job.py @@ -0,0 +1,200 @@ +import importlib +import inspect +import json +import os +import sys + +from tqdm import tqdm + +from evoflow.controller.log_controller import logger, pretty_dict +from evoflow.entities.core.base_object import BaseObject +from evoflow.entities.core.step import Step +from evoflow.entities.core.transaction import Transaction + + +class Job(BaseObject): + """ + Job : Các bot sẽ được tạo dưới dạng các Job. + + Example: + job = Job(name='#323', steps=[ + ReadConfigFile(config_file_path='config/process_a1_config.xlsx'), + PrepareInput(), + RemoveUnusedPart(), + GetDistanceThreshold(), + MainProcess(), + ShowHideForCapture(), + Capture(), + SaveOutput(), + ]) + + job.run() # run bot + job.package('bots') # package bot + + """ + + def __info__(self, **kwargs) -> dict: + info = f"Job {self.name}" + return info + + def kill(self, **kwargs): + pass + + def finish(self, **kwargs): + logger.info(f"Finish job: {self.name}") + + def __step_generator(self): + self.stacks = [self.start_step] + while len(self.stacks) > 0: + step_i = self.stacks.pop() + self.current_step = step_i + yield step_i + + def run(self, **kwargs): + logger.info(f"Running job: {self.name}") + self.params_pool = kwargs + step_generator = self.__step_generator() + + for step in tqdm(step_generator, unit="step"): + log_string = f"Running step : {step.name}" + logger.info(log_string) + + step.prepare(**self.params_pool) + try: + action_params = inspect.getfullargspec(step.action).args + build_params = {} + for param in action_params: + if param == "self": + continue + build_params[param] = step.__dict__.get(param) + last_result = step.action(**build_params) + except AttributeError: + logger.error(f"Current Job params: {pretty_dict(self.params_pool)}") + raise + step.end(**kwargs) + + if last_result is not None: + self.params_pool = {**self.params_pool, **last_result} + step.set_all_params(self.params_pool) + self.stacks += step.get_next_steps(self.params_pool) + + self.finish() + + return last_result + + def __init__(self, name=None, start_step: Step = None, **kwargs): + self.current_step = None + self.__start_step = start_step + self.params_pool = {} + if name is None: + name = os.getenv("JOB_NAME") + super().__init__(name=name, **kwargs) + + @property + def start_step(self): + return self.__start_step + + @start_step.setter + def start_step(self, value): + self.__start_step = value + self.__start_step.is_start = True + + def document(self): + pass + + def show_graph(self): + pass + + def get_all_steps(self): + steps = [self.start_step] + queues = [self.start_step] + while queues: + step = queues.pop() + transactions = step.transactions + for transaction in transactions: + transaction: Transaction + next_step = transaction.to + if next_step not in steps: + steps.append(next_step) + queues.append(next_step) + return steps + + def to_nodered_id(self, id): + return id[:8] + "." + id[8:13] + + def to_json(self): + # jsonStr = json.dumps(obj.__dict__) + steps = [self.start_step] + steps.extend(self.get_all_steps()) + steps_dict = [ + { + "id": "7027fc3.86fe004", + "type": "tab", + "label": "Flow 1", + "disabled": False, + "info": "", + } + ] + for i in range(len(steps)): + step_dict = {} + step_dict["id"] = self.to_nodered_id(steps[i].id) + step_dict["type"] = "step" + step_dict["z"] = "7027fc3.86fe004" + step_dict["name"] = "Step" + str(i + 1) + params_dict = {} + params = inspect.getfullargspec(steps[i].action).args + for param in params: + params_dict[param] = steps[i].__dict__.get(param) + step_dict["params"] = params_dict + condition = "" + for step in steps: + if steps[i] == step: + continue + for transaction in step.transactions: + if transaction.to.id == steps[i].id: + condition = transaction.condition.condition_function + break + step_dict["condition"] = condition + step_dict["conditionFormat"] = "string" + step_dict["stepType"] = steps[i].action.__name__ + step_dict["x"] = 100 + step_dict["y"] = 50 + i * 50 + # get transactions + transactions_dict = [] + for transaction in steps[i].transactions: + transactions_dict.append(self.to_nodered_id(transaction.to.id)) + step_dict["wires"] = transactions_dict + steps_dict.append(step_dict) + return json.dumps(steps_dict, ensure_ascii=False, indent=4) + + @staticmethod + def load_job(path: str): + base_path, module = os.path.split(path) + sys.path.append(base_path) + job = importlib.import_module(f"{module}.main").job + return job + + def __call__(self, original_function): + def wrapper_job(*args, **kwargs): + step = Step(*args, **kwargs) + step.action = original_function + self.start_step = step + return self + + return wrapper_job + + def to_dict(self): + steps = self.get_all_steps() + data = {"name": self.name, "id": self.id} + data["steps"] = [] + for step in steps: + step: Step + data["steps"].append(step.to_dict()) + + return data + + def __enter__(self, *args, **kwargs): + return self + + def __exit__(self, *args, **kwargs): + self.finish() diff --git a/evoflow/entities/core/session.py b/evoflow/entities/core/session.py new file mode 100644 index 0000000..0b6cae4 --- /dev/null +++ b/evoflow/entities/core/session.py @@ -0,0 +1,10 @@ + +class Session: + def __init__(self): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass \ No newline at end of file diff --git a/evoflow/entities/core/step.py b/evoflow/entities/core/step.py new file mode 100644 index 0000000..a375327 --- /dev/null +++ b/evoflow/entities/core/step.py @@ -0,0 +1,122 @@ +import abc +import inspect +from copy import deepcopy +from functools import wraps + +from evoflow.entities.core.base_object import BaseObject + +from collections import OrderedDict + + +class Step(BaseObject): + """ + Step is a cells of jobs + """ + + def __call__(self, func) -> "Step": + """ + + :return: Step object + :rtype: Step + """ + + def step_decorator(func): + @wraps(func) + def wrapper_action(*args: tuple, **kwargs: dict) -> "Step": + """ + Wrapper method để tạo thành action trong step + :param args: + :param kwargs: + :return: + :rtype: Step + """ + self.name = func.__name__.replace("_", " ").capitalize() + params = inspect.getfullargspec(func) + kwargs = {**kwargs, **dict(zip(list(params.args), args))} + self.prepare(**kwargs) + self.action = func + return deepcopy(self) + + return wrapper_action + + return step_decorator(func) + + def __init__(self, name=None, transactions=None, **kwargs): + """ + + @param name: Name of the step + @param kwargs: + """ + self.is_start = kwargs.get("is_start") + self.is_end = kwargs.get("is_end") + if transactions is None: + self.transactions = [] + else: + self.transactions = transactions + + if name is None: + name = self.__class__.__name__ + + self.params = {"name": name} + super().__init__(name=name, **kwargs) + + @abc.abstractmethod + def action(self, **kwargs): + """ + Performs the function of step + """ + pass + + def end(self, **kwargs) -> dict: + """ + Kết thúc step, kill các object không cần thiết để giải phóng bộ nhớ + """ + # Global.caa.start_command('clear history') + + def prepare(self, **kwargs): + """ + Chuẩn bị cho action, hàm prepare giúp chuẩn bị input để đẩy vào action() + """ + for k, v in kwargs.items(): + setattr(self, k, v) + self.params = kwargs + + def __str__(self): + return self.name + + def get_next_steps(self, params=None): + next_steps = [] + + for transaction in self.transactions: + if transaction.validate(**params): + next_steps.append(transaction.to) + + return next_steps + + def next(self, to, condition="always"): + from evoflow.entities.core.transaction import Transaction + + Transaction(step=self, to=to, condition=condition) + return to + + def set_all_params(self, params): + self.params = params + + # support '>>' operator + def __rshift__(self, other): + from evoflow.entities.core.step_list import StepList + # connect to other step + if isinstance(other, Step) or isinstance(other, StepList): + return self.next(other) + elif isinstance(other, list): + return self.next(StepList(other)) + + def __lshift__(self, other): + + from evoflow.entities.core.step_list import StepList + # connect to other step + if isinstance(other, Step) or isinstance(other, StepList): + return other.next(self) + elif isinstance(other, list): + step_list = StepList(other) + return step_list.next(self) diff --git a/evoflow/entities/core/step_list.py b/evoflow/entities/core/step_list.py new file mode 100644 index 0000000..eb0e1b0 --- /dev/null +++ b/evoflow/entities/core/step_list.py @@ -0,0 +1,56 @@ +import typing +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from evoflow.entities.core.step import Step + + +class StepList(Step): + def __init__(self, steps=None, transactions=None, **kwargs): + super().__init__(**kwargs) + self.steps = steps or [] + self.transactions = transactions or [] + + def add_step(self, step: "Step"): + self.steps.append(step) + + def __rshift__(self, other): + """ + @param other: Step + @return: + + """ + from evoflow.entities.core.step import Step + + if isinstance(other, Step): + return self.next(other) + raise TypeError(f"StepList can't connect to {type(other)}") + + def next(self, to: typing.Union["Step", "StepList"], condition="always"): + from evoflow.entities.core.transaction import Transaction + + Transaction(step=self, to=to, condition=condition) + + return to + + def __repr__(self): + return f"StepList: {self.steps}" + + def __str__(self): + return f"StepList: {self.steps}" + + def __getitem__(self, item): + return self.steps[item] + + def action(self, **kwargs): + """ + Performs the function of step + """ + kwargs = {**self.params, **kwargs} + with ThreadPoolExecutor(max_workers=10) as executor: + list( + tqdm( + executor.map(lambda step: step.action(**kwargs), self.steps), total=len(self.steps), desc=self.name + ) + ) diff --git a/evoflow/entities/core/transaction.py b/evoflow/entities/core/transaction.py new file mode 100644 index 0000000..c30b101 --- /dev/null +++ b/evoflow/entities/core/transaction.py @@ -0,0 +1,41 @@ +from evoflow import logger +from evoflow.entities.core.base_object import BaseObject +from evoflow.entities.core.condition import Condition +from evoflow.entities.core.step import Step + + +class Transaction(BaseObject): + def __init__(self, step: Step, to: Step, condition="always"): + self.id = str(step.id) + "-" + str(to.id) + self.to = to + self.__from_step__ = step + self.condition = Condition(condition) + self.__from_step__.transactions.append(self) + + def validate(self, **kwargs): + if type(self.condition.condition_function) == str: + if self.condition.condition_function == "always": + return True + return self.build_condition_from_string(self.condition.condition_function, self.__from_step__.params) + return bool(self.condition.condition_function(**kwargs)) + + def build_condition_from_string(self, condition_function_string, params): + """ + @param condition_function_string: Ex - select_condition == 3 + @return: + """ + for k, v in params.items(): + try: + exec(f"{k} = {v}\n") + except Exception as e: + logger.debug( + f"{e}\nCan't build param {k} to create condition from string, try to create condition with lambda function" + ) + exec("global result\nresult = %s" % (condition_function_string)) + global result + return result + + def to_dict(self): + data = {"id": self.id, "from": self.__from_step__.id, "to": self.to.id, "condition": self.condition} + + return data diff --git a/evoflow/entities/core/workflow.py b/evoflow/entities/core/workflow.py new file mode 100644 index 0000000..e7e769e --- /dev/null +++ b/evoflow/entities/core/workflow.py @@ -0,0 +1,26 @@ +from evoflow.entities.core.base_object import BaseObject + + +class WorkFlow(BaseObject): + def kill(self, **kwargs): + pass + + def __info__(self, **kwargs) -> dict: + current_job_index = self.jobs.index(self.current_job) + info = { + 'jobs': len(self.jobs), + 'current job': f'{current_job_index} - {self.current_job.name}' + } + + def __init__(self, name, jobs: list = [], **kwargs): + super(WorkFlow, self).__init__(name=name, **kwargs) + self.jobs = jobs + self.current_job = self.jobs[0] + + def __dict__(self): + data = { + 'jobs': [job.__dict__() for job in self.jobs], + 'current_job': self.current_job.name, + } + + return data diff --git a/evoflow/entities/data_manipulate/__init__.py b/evoflow/entities/data_manipulate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/data_manipulate/abstract_data.py b/evoflow/entities/data_manipulate/abstract_data.py new file mode 100644 index 0000000..8ca2b02 --- /dev/null +++ b/evoflow/entities/data_manipulate/abstract_data.py @@ -0,0 +1,34 @@ +import abc + +import json_tricks as json + + +class AbstractData: + def __init__(self): + pass + + @abc.abstractmethod + def get_info(self) -> str: + """ + print một đoạn mô tả về data object này + :return: + """ + info = {'info': "Abstract data"} + return info + + def summary(self): + """ + Summary info of current object + """ + info = self.get_info() + return json.dumps(info, ensure_ascii=False, indent=2) + + @abc.abstractmethod + def save(self, file_path, **args) -> str: + """ + Lưu data này lại + :param args: + :return: đường dẫn tới file sau khi save ra + """ + + raise NotImplementedError() diff --git a/evoflow/entities/data_manipulate/file_operator/.gitkeep b/evoflow/entities/data_manipulate/file_operator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/data_manipulate/file_operator/__init__.py b/evoflow/entities/data_manipulate/file_operator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/data_manipulate/file_operator/dataframe_file.py b/evoflow/entities/data_manipulate/file_operator/dataframe_file.py new file mode 100644 index 0000000..21381e4 --- /dev/null +++ b/evoflow/entities/data_manipulate/file_operator/dataframe_file.py @@ -0,0 +1,47 @@ +import pandas as pd +from openpyxl import load_workbook + +from evoflow.entities.data_manipulate.file_operator.file import File + + +class DataFrameFile(File): + """ + Handle pandas DataFrame files + """ + + def save(self, file_path=None) -> bool: + # pylint: disable=abstract-class-instantiated + writer = pd.ExcelWriter(file_path, engine='xlsxwriter') + + # skip first row + self.data.to_excel(writer, sheet_name='Sheet1', startrow=1, header=False, index=False) + # pylint: disable=no-member + workbook = writer.book + worksheet = writer.sheets['Sheet1'] + # Add a header format. + header_format = workbook.add_format({ + 'bold': True, + 'fg_color': '#03a5fc', + 'border': 1}) + + for col_num, value in enumerate(self.data.columns.values): + # write to second row + worksheet.write(0, col_num, value, header_format) + column_len = self.data[value].astype(str).str.len().max() + column_len = max(column_len, len(value)) + worksheet.set_column(col_num, col_num, column_len) + writer.save() + return True + + # def get_info(self) -> str: + # wb = load_workbook(self.file_path) + # properties = wb.properties + # return self.data.head() + + def __init__(self, **args): + super().__init__(**args) + self.data: pd.DataFrame + if self.file_path is not None: + self.__meta_data__ = load_workbook(self.file_path).properties + else: + self.__meta_data__ = None diff --git a/evoflow/entities/data_manipulate/file_operator/file.py b/evoflow/entities/data_manipulate/file_operator/file.py new file mode 100644 index 0000000..946abdf --- /dev/null +++ b/evoflow/entities/data_manipulate/file_operator/file.py @@ -0,0 +1,46 @@ +import os +from abc import abstractmethod + +from evoflow.entities.data_manipulate.abstract_data import AbstractData + + +class File(AbstractData): + def __init__(self, file_path: str = None, **args): + self.file_path = file_path + + self.file_name = os.path.split(file_path)[1].replace('.' + self.get_file_type(), '') + self.data = args.get('data') + self.__meta_data__ = None + super().__init__() + + def get_file_type(self) -> str: + """ + Trả về loại file dựa vào đuôi của file + :return: Loại file + """ + + return os.path.split(self.file_path)[1].split(".")[1] + + def get_info(self) -> str: + meta_data = self.__meta_data__ if self.__meta_data__ else self.data + attr_names = [z for z in dir(meta_data) if not z.startswith('_')] + info = {} + for attr_name in attr_names: + value = getattr(meta_data, attr_name) + if value is not None \ + and value.__class__.__name__ in ['int', 'str', 'datetime'] \ + and len(str(value)) > 0: + if value.__class__.__name__ in ['int', 'str']: + info[attr_name] = value + else: + info[attr_name] = str(value) + return info + + @abstractmethod + def save(self, file_path=None, **args) -> bool: + """ + Save file vào file_path + :file_path: đường dẫn output file, + nếu file_path là None thì tự tạo tên file và save vào cache + :return: trả về đường dẫn tới file + """ diff --git a/evoflow/entities/data_manipulate/file_operator/image_file.py b/evoflow/entities/data_manipulate/file_operator/image_file.py new file mode 100644 index 0000000..9ec400a --- /dev/null +++ b/evoflow/entities/data_manipulate/file_operator/image_file.py @@ -0,0 +1,67 @@ +import os + +import cv2 +import PIL +from PIL import Image, ImageDraw, ImageFont + +from evoflow.entities.data_manipulate.file_operator.file import File +from evoflow.entities.Global import Global +from evoflow.Services.OCR.EasyOCREngine import EasyOCREngine +from evoflow.Services.OCR.Result import OCRResult + + +class ImageFile(File): + """ + Read and write image files + """ + + def save(self, file_path=None) -> str: + cv2.imwrite(file_path, self.data) + return file_path + + def get_info(self) -> str: + image_name = self.file_path.split(os.sep)[-1] + image_shape = self.data.shape + return {'image name': image_name, 'image_shape': image_shape} + + def get_texts(self, languages=None, **kwargs): + """ + try to ocr text from Image + """ + if languages is None: + languages = ['en'] + if Global.ocr_engine is None: + Global.ocr_engine = EasyOCREngine(languages=languages) + res = Global.ocr_engine.ocr(self.data, detail=1) + return res + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def to_pil(self): + return PIL.Image.fromarray(self.data) + + def draw(self, ocr_results): + user_path = f'{os.getenv("userprofile")}/.evoflow/fonts/Noto_Sans_JP/NotoSansJP-Regular.otf' + data_path = './data/.evoflow/fonts/Noto_Sans_JP/NotoSansJP-Regular.otf' + + if os.path.isfile(user_path): + font_file = user_path + else: + font_file = data_path + + font = ImageFont.truetype(font_file, 14) + mat = self.data + if len(mat.shape) == 2: + mat = cv2.cvtColor(mat, cv2.COLOR_GRAY2RGB) + img = Image.fromarray(mat) + drawer = ImageDraw.Draw(img) + text_color = 'red' + for ocr_result in ocr_results: + ocr_result: OCRResult + top_left = ocr_result.bbox.xmin, ocr_result.bbox.ymin + bbox = ocr_result.bbox.xmin, ocr_result.bbox.ymin, \ + ocr_result.bbox.xmax, ocr_result.bbox.ymax + drawer.text(top_left, ocr_result.text, fill=text_color, font=font) + drawer.rectangle(bbox, outline='green') + return img diff --git a/evoflow/entities/data_manipulate/file_operator/pdf_file.py b/evoflow/entities/data_manipulate/file_operator/pdf_file.py new file mode 100644 index 0000000..ef13e32 --- /dev/null +++ b/evoflow/entities/data_manipulate/file_operator/pdf_file.py @@ -0,0 +1,99 @@ +import glob +import json +import os +import pathlib +import urllib +from typing import Iterator + +import cv2 +import numpy as np +from tqdm import tqdm + +import evoflow.Params +from evoflow import logger +from evoflow.controller.data_manipulate.image_file_operator import ImageFileOperator +from evoflow.entities.data_manipulate.file_operator.file import File +from evoflow.entities.data_manipulate.file_operator.image_file import ImageFile + + +def download_poppler(): + try: + from pyunpack import Archive + except ImportError: + logger.error("Can't import pyunpack. Try to install with:\npip install pyunpack") + + poppler_path = f"{os.getenv('userprofile')}/.evoflow/poppler" + pathlib.Path(poppler_path).mkdir(parents=True, exist_ok=True) + poppler_file_name = evoflow.Params.POPPLER_URL.rsplit('/', maxsplit=-1) + opener = urllib.request.URLopener() + opener.addheader('User-Agent', 'evoflow') + filename, _ = opener.retrieve(evoflow.Params.POPPLER_URL, f"{poppler_path}/{poppler_file_name}") + Archive(filename).extractall(poppler_path) + os.remove(filename) + poppler_path = glob.glob(f'{poppler_path}/*/bin')[0] + return os.path.normpath(poppler_path) + + +class PdfFile(File): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def get_texts(self): + page_count = len(self.data.pages) + texts = [] + for i in range(page_count): + page = self.data.pages[i] + text = page.extract_text() + # text = page_content.encode('utf-8') + texts.append(text) + return texts + + def get_info(self) -> str: + return json.dumps(self.data.metadata, ensure_ascii=False, indent=2) + + def to_images(self, dpi=500) -> Iterator[ImageFile]: + + try: + # pylint: disable= import-outdide-toplevel + from pdf2image import convert_from_path + except ImportError as import_error: + raise ImportError("Can't import pdf2image. Try to install with: pip install pdf2image") from import_error + + poppler_paths = glob.glob(f"{os.getenv('userprofile')}/.evoflow/poppler/*/bin") + if len(poppler_paths) == 0: + try: + poppler_path = download_poppler() + except ValueError as value_error: + if str(value_error) == 'patool not found! Please install patool!': + raise ValueError( + 'patool not found! Please install patool!. \n' + 'Try to install with: pip install patool') from value_error + else: + poppler_path = poppler_paths[0] + + pages = convert_from_path(self.file_path, dpi, poppler_path=poppler_path) + images = [] + for i, page in tqdm(enumerate(pages), + total=len(pages), + desc=f'Extract image from file: {self.file_path}'): + image_path = f"{self.file_path}_page_{i}.png" + page.save(image_path) + image = ImageFileOperator().read(image_path) + os.remove(image_path) + images.append(image) + return images + + def to_full_image(self, dpi=200) -> ImageFile: + images = self.to_images(dpi=dpi) + full_image = images.pop(0) + for image in images: + mat1 = full_image.data + mat2 = image.data + h_1, w_1 = mat1.shape[:2] + _, w_2 = mat2.shape[:2] + + if w_1 != w_2: + mat2 = cv2.resize(mat2, (w_1, int(h_1 * (w_1 / w_2)))) + + full_image.data = np.concatenate([mat1, mat2], axis=0) + return full_image diff --git a/evoflow/entities/data_manipulate/file_operator/pptx_file.py b/evoflow/entities/data_manipulate/file_operator/pptx_file.py new file mode 100644 index 0000000..f198680 --- /dev/null +++ b/evoflow/entities/data_manipulate/file_operator/pptx_file.py @@ -0,0 +1,76 @@ +import json +import pathlib + +from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.presentation import Presentation +from tqdm import tqdm + +from evoflow.entities.data_manipulate.file_operator.file import File + + +def iter_picture_shapes(prs): + for slide in prs.slides: + for shape in slide.shapes: + # pylint: disable=no-member + if shape.shape_type == MSO_SHAPE_TYPE.PICTURE: + yield shape + + +class PPTXFile(File): + def save(self, file_path=None) -> str: + self.data.save(file_path) + + def get_info(self) -> str: + self.data: Presentation + + properties = [] + for attribute in dir(self.data.core_properties): + if not attribute.startswith('_'): + properties.append(attribute) + + properties_dict = {'slides': len(self.data.slides)} + for property_name in properties: + + value = getattr(self.data.core_properties, property_name) + + if value is not None \ + and value.__class__.__name__ in ['int', 'str', 'datetime'] \ + and len(str(value)) > 0: + properties_dict[property_name] = str(value) + + return json.dumps(properties_dict, ensure_ascii=False, indent=2) + + def extract_images(self, output_path='data'): + index = 0 + for picture in tqdm(iter_picture_shapes(self.data), desc='Extract Images'): + image = picture.image + # ---get image "file" contents--- + image_bytes = image.blob + # ---make up a name for the file, e.g. 'image.jpg'--- + + image_dir = f'{output_path}/{self.data.core_properties.title}' + image_filename = f'{image_dir}/image_{index}.{image.ext}' + + pathlib.Path(image_dir).mkdir(parents=True, exist_ok=True) + + with open(image_filename, 'wb') as file: + file.write(image_bytes) + index += 1 + + def get_texts(self): + prs = self.data + text_runs = [] + + for slide in prs.slides: + for shape in slide.shapes: + if not shape.has_text_frame: + continue + for paragraph in shape.text_frame.paragraphs: + for run in paragraph.runs: + text_runs.append(run.text) + + return text_runs + + def __init__(self, **args): + super().__init__(**args) + self.data: Presentation diff --git a/evoflow/entities/exceptions/__init__.py b/evoflow/entities/exceptions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/entities/exceptions/base_exception.py b/evoflow/entities/exceptions/base_exception.py new file mode 100644 index 0000000..b890da4 --- /dev/null +++ b/evoflow/entities/exceptions/base_exception.py @@ -0,0 +1,6 @@ + + + +class BaseException(Exception): + """Base exception class for all exceptions in this project.""" + pass \ No newline at end of file diff --git a/evoflow/entities/exceptions/evo_exception.py b/evoflow/entities/exceptions/evo_exception.py new file mode 100644 index 0000000..f8053a9 --- /dev/null +++ b/evoflow/entities/exceptions/evo_exception.py @@ -0,0 +1,5 @@ + + +class EvoException(BaseException): + """Base exception class for all exceptions in this project.""" + pass \ No newline at end of file diff --git a/evoflow/entities/global.py b/evoflow/entities/global.py new file mode 100644 index 0000000..6910e12 --- /dev/null +++ b/evoflow/entities/global.py @@ -0,0 +1,39 @@ +import os + +from evoflow import logger + +try: + from pywinauto import Application as App +except ImportError: + logger.debug("Can't import pywin32, try to install with:\nconda install pywin32\nOR\npip install pywin32==227") + +from evoflow.controller.log_controller import logger + +DATA_PATH = 'data' +START_CATIA = True + + + + +class Global(): + REMOTE_EXECUTE = False + __caa = None + custom_env = None + ocr_engine = None + + def set_env(self, env): + Global.custom_env = env + return Global.custom_env + + @property + def caa(self): + if Global.__caa is None: + try: + logger.info('Opening CATIA Application ... ') + if Global.custom_env is not None: + Global.__caa = start_catia(Global.custom_env) + else: + Global.__caa = start_catia(None) + except: + logger.error("Can't start CATIA") + return Global.__caa diff --git a/evoflow/operators/__init__.py b/evoflow/operators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evoflow/operators/base_operator.py b/evoflow/operators/base_operator.py new file mode 100644 index 0000000..732e7d4 --- /dev/null +++ b/evoflow/operators/base_operator.py @@ -0,0 +1,13 @@ +from evoflow import Step + +class BaseOperator(Step): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.task_id = kwargs.get('task_id', None) + self.name = self.task_id + + def __str__(self): + return f"{self.__class__.__name__} {self.task_id}" + + def __repr__(self): + return f"{self.__class__.__name__} {self.task_id}" \ No newline at end of file diff --git a/evoflow/operators/empty_operator.py b/evoflow/operators/empty_operator.py new file mode 100644 index 0000000..d3b132e --- /dev/null +++ b/evoflow/operators/empty_operator.py @@ -0,0 +1,27 @@ +from evoflow.operators.base_operator import BaseOperator + + +class EmptyOperator(BaseOperator): + """ + Executes a Python callable + :param python_callable: A reference to an object that is callable + :type python_callable: python callable + :param op_args: a list of positional arguments to pass to python_callable + :type op_args: list + :param op_kwargs: a dict of keyword arguments to pass to python_callable + :type op_kwargs: dict + :param templates_dict: a dictionary of templates variables to pass to python_callable + :type templates_dict: dict + :param templates_exts: a list of file extensions to resolve templates + :type templates_exts: list + """ + template_fields = ('templates_dict', 'op_args', 'op_kwargs') + template_ext = tuple() + ui_color = '#ffefeb' + + def __init__( + self, + + *args, **kwargs): + super().__init__(*args, **kwargs) + \ No newline at end of file diff --git a/evoflow/operators/python_operator.py b/evoflow/operators/python_operator.py new file mode 100644 index 0000000..dd9970a --- /dev/null +++ b/evoflow/operators/python_operator.py @@ -0,0 +1,41 @@ + +from evoflow.operators.base_operator import BaseOperator + + +class PythonOperator(BaseOperator): + """ + Executes a Python callable + :param python_callable: A reference to an object that is callable + :type python_callable: python callable + :param op_args: a list of positional arguments to pass to python_callable + :type op_args: list + :param op_kwargs: a dict of keyword arguments to pass to python_callable + :type op_kwargs: dict + :param templates_dict: a dictionary of templates variables to pass to python_callable + :type templates_dict: dict + :param templates_exts: a list of file extensions to resolve templates + :type templates_exts: list + """ + template_fields = ('templates_dict', 'op_args', 'op_kwargs') + template_ext = tuple() + ui_color = '#ffefeb' + + def __init__( + self, + python_callable, + op_args=None, + op_kwargs=None, + templates_dict=None, + templates_exts=None, + *args, **kwargs): + super().__init__(*args, **kwargs) + self.python_callable = python_callable + self.op_args = op_args or [] + self.op_kwargs = op_kwargs or {} + self.templates_dict = templates_dict + self.templates_exts = templates_exts + + def action(self, **kwargs): + kwargs = {**kwargs, **self.op_kwargs} + return self.python_callable(*self.op_args, **kwargs) + diff --git a/evoflow/params.py b/evoflow/params.py new file mode 100644 index 0000000..1eacb49 --- /dev/null +++ b/evoflow/params.py @@ -0,0 +1,2 @@ +POPPLER_URL = "http://blog.alivate.com.au/wp-content/uploads/2018/10/poppler-0.68.0_x86.7z" +UPX_URL = "https://github.com/upx/upx/releases/download/v3.96/upx-3.96-win64.zip" diff --git a/evoflow/version.py b/evoflow/version.py new file mode 100644 index 0000000..6e6c9c6 --- /dev/null +++ b/evoflow/version.py @@ -0,0 +1,4 @@ +import os + +root = os.path.dirname(os.path.abspath(__file__)) +__version__ = open(os.path.join(root, 'VERSION')).read().strip() \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 676d5a0..6bf9fb7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,2 +1,2 @@ -site_name: evo_flow +site_name: evoflow theme: readthedocs diff --git a/setup.py b/setup.py index 960e88e..9c33efe 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ def read(*paths, **kwargs): """Read the contents of a text file safely. - >>> read("evo_flow", "VERSION") + >>> read("evoflow", "VERSION") '0.1.0' >>> read("README.md") ... @@ -30,9 +30,9 @@ def read_requirements(path): setup( - name="evo_flow", - version=read("evo_flow", "VERSION"), - description="Awesome evo_flow created by maycuatroi", + name="evoflow", + version=read("evoflow", "VERSION"), + description="Awesome evoflow created by maycuatroi", url="https://github.com/maycuatroi/evo-flow/", long_description=read("README.md"), long_description_content_type="text/markdown", @@ -40,7 +40,7 @@ def read_requirements(path): packages=find_packages(exclude=["tests", ".github"]), install_requires=read_requirements("requirements.txt"), entry_points={ - "console_scripts": ["evo_flow = evo_flow.__main__:main"] + "console_scripts": ["evoflow = evoflow.__main__:main"] }, extras_require={"test": read_requirements("requirements-test.txt")}, ) diff --git a/tests/test_base.py b/tests/test_base.py index 20e344c..29eb9f5 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,5 +1,5 @@ -from evo_flow.base import NAME +from evoflow.base import NAME def test_base(): - assert NAME == "evo_flow" + assert NAME == "evoflow"