diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bf2b932 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +--- +name: Build and test + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install mypy flake8 types-termcolor==1.1.6 + - name: Type-check + run: | + mypy src/ tests/ examples/ + - name: Linter + run: | + flake8 --max-line-length=100 + - name: Build package + run: | + python -m build + - uses: actions/upload-artifact@v3 + with: + name: cloudproof_py_dist + path: ./dist + retention-days: 1 + + test: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: cloudproof_py_dist + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cloudproof_py*.whl + - name: Run tests + run: | + python -m unittest tests/test*.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..0f33142 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,55 @@ +name: Publish + +on: + push: + tags: + - '*' + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Release + uses: softprops/action-gh-release@v1 + + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Install dependencies and build + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install twine + python -m build + - name: Upload to Pypi + run: twine upload -u "${PYPI_USERNAME}" -p "${PYPI_PASSWORD}" dist/cloudproof_py*.whl + env: + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r docs/requirements.txt + - name: Build package + run: | + python scripts/extract_lib_types.py + cd docs && make html + - uses: actions/upload-artifact@v3 + with: + name: html_doc + path: docs/_build/html + retention-days: 60 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6a4ef11 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,63 @@ +--- +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-byte-order-marker + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key +# - id: double-quote-string-fixer + - id: end-of-file-fixer + - id: file-contents-sorter + - id: fix-byte-order-marker + - id: fix-encoding-pragma + - id: mixed-line-ending + args: [--fix=lf] + - id: requirements-txt-fixer + - id: sort-simple-yaml + - id: trailing-whitespace + + - repo: https://github.com/compilerla/conventional-pre-commit + rev: v2.1.1 + hooks: + - id: conventional-pre-commit + stages: [commit-msg] + args: [] # optional: list of Conventional Commits types to allow e.g. [feat, fix, ci, chore, test] + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0-alpha.4 + hooks: + - id: prettier + types_or: [css, javascript, jsx, markdown, bash, java, sh] + + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + args: [--max-line-length=100] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.990' + hooks: + - id: mypy + additional_dependencies: [types_termcolor==1.1.6] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0f42e40 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [1.0.0] - 2022-12-21 + +### Documentation + +- Add sphinx support for automatic doc generation + +### Features + +- Add package setup +- Add abstract class around FindexInternal +- Add sqlite example for upsert and search +- Add sub-word graph generation util function for Findex +- Make an interactive cli demo + +### Testing + +- Add SQLite findex tests +- Add CoverCrypt tests + +### Ci + +- Add automatic package build in github workflow +- Add automatic package publishing in github workflow + + diff --git a/README.md b/README.md index 76d05a6..978bb71 100644 --- a/README.md +++ b/README.md @@ -1 +1,43 @@ -

Cloudproof Encryption Python Library

+# Cloudproof Encryption Python Library + +![ci status](https://github.com/Cosmian/cloudproof_python/actions/workflows/ci.yml/badge.svg) + +The library provides a Python API to the **Cloudproof Encryption** product of the [Cosmian Ubiquitous Encryption platform](https://cosmian.com). + +Please [check the online documentation](https://docs.cosmian.com/cloudproof_encryption/use_cases_benefits/) for details on using the CloudProof APIs. + +## Build package + +```sh +pip install -r requirements.txt +scripts/build.sh [-i] [-t] +``` + +## Build docs + +```sh +pip install -r docs/requirements.txt +scripts/build.sh -d +``` + +## Demo + +An interactive CLI demo combining policy-based encryption with searchable keywords. + +Users from `./examples/cli_demo/data.json` are encrypted using CoverCrypt and indexed via Findex. + +- Run + +```sh +scripts/run_demo.sh +``` + +## Versions Correspondence + +This library depends on [CoverCrypt](https://github.com/Cosmian/cover_crypt) and [Findex](https://github.com/Cosmian/findex). + +This table shows the minimum version correspondence between the various components. + +| `cloudproof_py` | CoverCrypt | Findex | +| --------------- | ---------- | ------ | +| 1.0.0 | 8.0.1 | 1.0.1 | diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..4befcae --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "CloudProof_Py" +copyright = "2022, Cosmian Tech" +author = "Cosmian Tech" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx_rtd_theme", "sphinx.ext.napoleon", "autoapi.extension"] +autoapi_type = "python" +autoapi_dirs = ["../src"] +# Use py interface files in priority +autoapi_file_patterns = ["*.docpy", "*.py"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..10b67c2 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,5 @@ +Welcome to CloudProof Python's documentation +============================================ + +.. toctree:: + :maxdepth: 4 diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..a106df3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx-autoapi>=2.0.0 +sphinx-rtd-theme>=1.1.1 diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..fd3ec55 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,5 @@ +# Examples + +This folder contains examples to get started with `Cloudproof Python`: + +- Demo combining encryption (CoverCrypt) and search (Findex): [cli_demo](./cli_demo) diff --git a/examples/cli_demo/data.json b/examples/cli_demo/data.json new file mode 100644 index 0000000..2ffe873 --- /dev/null +++ b/examples/cli_demo/data.json @@ -0,0 +1,1002 @@ +[ + { + "firstName": "Felix", + "lastName": "Shepherd", + "phone": "06 52 23 63 25", + "email": "orci@icloud.org", + "country": "Germany", + "region": "Upper Austria", + "employeeNumber": "SPN82TTO0PP", + "security": "confidential" + }, + { + "firstName": "Emerson", + "lastName": "Wilkins", + "phone": "01 01 31 41 37", + "email": "enim.diam@icloud.edu", + "country": "Spain", + "region": "Antofagasta", + "employeeNumber": "BYE60HQT6XG", + "security": "confidential" + }, + { + "firstName": "Ocean", + "lastName": "Meyers", + "phone": "07 45 55 66 55", + "email": "ultrices.vivamus@aol.net", + "country": "Spain", + "region": "Podlaskie", + "employeeNumber": "SXK82FCR9EP", + "security": "confidential" + }, + { + "firstName": "Kiara", + "lastName": "Harper", + "phone": "07 17 88 69 58", + "email": "vitae@outlook.com", + "country": "Germany", + "region": "Rajasthan", + "employeeNumber": "CWN36QTX2BN", + "security": "protected" + }, + { + "firstName": "Joelle", + "lastName": "Becker", + "phone": "01 11 46 84 14", + "email": "felis.adipiscing@hotmail.org", + "country": "France", + "region": "İzmir", + "employeeNumber": "AFR04EPJ1YM", + "security": "protected" + }, + { + "firstName": "Stacy", + "lastName": "Reyes", + "phone": "03 53 66 40 67", + "email": "risus.a@yahoo.ca", + "country": "France", + "region": "Nord-Pas-de-Calais", + "employeeNumber": "ZVW02EAM3ZC", + "security": "protected" + }, + { + "firstName": "Donna", + "lastName": "Velazquez", + "phone": "01 69 11 40 51", + "email": "mus.donec@aol.couk", + "country": "Germany", + "region": "Tuyên Quang", + "employeeNumber": "DOP17EIM7ST", + "security": "top_secret" + }, + { + "firstName": "Martin", + "lastName": "Snider", + "phone": "07 36 72 54 66", + "email": "fringilla.mi@google.com", + "country": "France", + "region": "Kansas", + "employeeNumber": "RYS34KBD5VW", + "security": "top_secret" + }, + { + "firstName": "Brielle", + "lastName": "Finley", + "phone": "02 75 95 77 31", + "email": "egestas.aliquam@hotmail.edu", + "country": "France", + "region": "Ulyanovsk Oblast", + "employeeNumber": "MFU36KUO6UD", + "security": "top_secret" + }, + { + "firstName": "Bryar", + "lastName": "Christian", + "phone": "02 44 54 20 55", + "email": "ullamcorper.eu.euismod@google.ca", + "country": "Germany", + "region": "Hatay", + "employeeNumber": "TRK72WOV9VH", + "security": "confidential" + }, + { + "firstName": "Diana", + "lastName": "Wilson", + "phone": "03 35 75 32 28", + "email": "nascetur.ridiculus.mus@outlook.net", + "country": "Germany", + "region": "Ulster", + "employeeNumber": "UPS23SOZ6QN", + "security": "confidential" + }, + { + "firstName": "Paul", + "lastName": "Ford", + "phone": "05 13 27 74 63", + "email": "pede.suspendisse@icloud.com", + "country": "Germany", + "region": "Rio Grande do Sul", + "employeeNumber": "CWT54TJX4RT", + "security": "confidential" + }, + { + "firstName": "Felicia", + "lastName": "Massey", + "phone": "06 72 81 43 63", + "email": "lacus.varius.et@yahoo.ca", + "country": "Germany", + "region": "Brecknockshire", + "employeeNumber": "BAC58KIS7DY", + "security": "protected" + }, + { + "firstName": "Barclay", + "lastName": "Allison", + "phone": "08 71 12 69 37", + "email": "in.cursus@aol.com", + "country": "Germany", + "region": "Caquetá", + "employeeNumber": "KLL08RGK2JW", + "security": "protected" + }, + { + "firstName": "Skyler", + "lastName": "Richmond", + "phone": "Figueroa", + "email": "elit@google.net", + "country": "France", + "region": "Chiapas", + "employeeNumber": "ITO71LVO4PD", + "security": "protected" + }, + { + "firstName": "Justin", + "lastName": "Cross", + "phone": "07 56 26 00 16", + "email": "neque.vitae@yahoo.edu", + "country": "Germany", + "region": "Friuli-Venezia Giulia", + "employeeNumber": "HHH01MIH6SZ", + "security": "top_secret" + }, + { + "firstName": "Miranda", + "lastName": "Cotton", + "phone": "06 73 42 44 47", + "email": "eget.magna@google.ca", + "country": "Spain", + "region": "Møre og Romsdal", + "employeeNumber": "DFW37PPI8TY", + "security": "top_secret" + }, + { + "firstName": "Figueroa", + "lastName": "Kane", + "phone": "02 44 08 45 32", + "email": "aenean.eget@protonmail.ca", + "country": "France", + "region": "Kansas", + "employeeNumber": "HFG82IKJ2OC", + "security": "top_secret" + }, + { + "firstName": "Lesley", + "lastName": "Sullivan", + "phone": "02 24 15 21 81", + "email": "orci.ut@protonmail.couk", + "country": "Spain", + "region": "Lubelskie", + "employeeNumber": "WOA67IVR6CM", + "security": "confidential" + }, + { + "firstName": "Clio", + "lastName": "Figueroa", + "phone": "06 87 82 58 97", + "email": "tellus@yahoo.org", + "country": "France", + "region": "Munster", + "employeeNumber": "ASS31LNB5CV", + "security": "confidential" + }, + { + "firstName": "Forrest", + "lastName": "Parsons", + "phone": "06 81 51 26 17", + "email": "iaculis.quis@yahoo.com", + "country": "France", + "region": "Hải Phòng", + "employeeNumber": "MJS53TBZ8UL", + "security": "confidential" + }, + { + "firstName": "Maxwell", + "lastName": "Park", + "phone": "01 37 79 08 31", + "email": "ut@yahoo.com", + "country": "Germany", + "region": "Guanacaste", + "employeeNumber": "PHQ43BNF8MI", + "security": "protected" + }, + { + "firstName": "Kalia", + "lastName": "Hayden", + "phone": "02 24 48 01 44", + "email": "non.egestas.a@aol.ca", + "country": "Spain", + "region": "Gävleborgs län", + "employeeNumber": "QUW73NPX4UG", + "security": "protected" + }, + { + "firstName": "Russell", + "lastName": "Willis", + "phone": "07 42 02 43 15", + "email": "sit.amet@icloud.edu", + "country": "France", + "region": "Bihar", + "employeeNumber": "QOU03UHS4LQ", + "security": "protected" + }, + { + "firstName": "Judah", + "lastName": "Chang", + "phone": "02 15 66 88 81", + "email": "tempor.arcu@icloud.ca", + "country": "Spain", + "region": "Assam", + "employeeNumber": "BJJ93AIN8LC", + "security": "top_secret" + }, + { + "firstName": "Chaim", + "lastName": "Richards", + "phone": "08 97 39 30 70", + "email": "nunc.sed@protonmail.com", + "country": "Germany", + "region": "Xīběi", + "employeeNumber": "JTS20MCR7GX", + "security": "top_secret" + }, + { + "firstName": "Zachary", + "lastName": "Porter", + "phone": "02 02 52 43 30", + "email": "suscipit.nonummy.fusce@hotmail.couk", + "country": "Germany", + "region": "Kaduna", + "employeeNumber": "NLN76SBS2EI", + "security": "top_secret" + }, + { + "firstName": "Cade", + "lastName": "Gould", + "phone": "09 82 18 22 16", + "email": "neque.vitae@google.org", + "country": "Germany", + "region": "Luik", + "employeeNumber": "RSW01YCJ6HJ", + "security": "confidential" + }, + { + "firstName": "Hiram", + "lastName": "Gates", + "phone": "02 46 85 81 87", + "email": "tristique.senectus@outlook.net", + "country": "Spain", + "region": "Niger", + "employeeNumber": "EGG84NJY5TH", + "security": "confidential" + }, + { + "firstName": "Deirdre", + "lastName": "Tate", + "phone": "02 99 26 61 08", + "email": "laoreet.ipsum@hotmail.com", + "country": "Spain", + "region": "Anambra", + "employeeNumber": "SLO36EYL1LQ", + "security": "confidential" + }, + { + "firstName": "Len", + "lastName": "Carlson", + "phone": "05 69 55 17 78", + "email": "non@aol.ca", + "country": "Germany", + "region": "Western Visayas", + "employeeNumber": "DHS57TIH5JX", + "security": "protected" + }, + { + "firstName": "Griffin", + "lastName": "Porter", + "phone": "03 44 78 02 98", + "email": "volutpat.nulla@protonmail.org", + "country": "Germany", + "region": "Rheinland-Pfalz", + "employeeNumber": "JTQ18TFU5XL", + "security": "protected" + }, + { + "firstName": "Caleb", + "lastName": "Sellers", + "phone": "08 43 33 76 76", + "email": "ipsum.dolor@aol.net", + "country": "Spain", + "region": "Lanarkshire", + "employeeNumber": "DJE23GBD4HV", + "security": "protected" + }, + { + "firstName": "Wang", + "lastName": "Chan", + "phone": "07 13 38 17 82", + "email": "venenatis.vel@outlook.net", + "country": "Spain", + "region": "Tripura", + "employeeNumber": "VYY77VOW0QR", + "security": "top_secret" + }, + { + "firstName": "Kalia", + "lastName": "Douglas", + "phone": "03 56 82 77 04", + "email": "mus.proin@hotmail.net", + "country": "France", + "region": "Tripura", + "employeeNumber": "AHM27UPN3HD", + "security": "top_secret" + }, + { + "firstName": "Ivy", + "lastName": "Wong", + "phone": "04 95 11 83 54", + "email": "sit.amet.lorem@google.org", + "country": "Germany", + "region": "Kahramanmaraş", + "employeeNumber": "YCE84QZN1AU", + "security": "top_secret" + }, + { + "firstName": "Brendan", + "lastName": "Rivers", + "phone": "02 24 31 11 26", + "email": "nonummy.ut@aol.couk", + "country": "Germany", + "region": "Free State", + "employeeNumber": "RVY06JHG6DN", + "security": "confidential" + }, + { + "firstName": "Jane", + "lastName": "Mckay", + "phone": "01 48 61 56 13", + "email": "vestibulum@protonmail.couk", + "country": "France", + "region": "Newfoundland and Labrador", + "employeeNumber": "CEL23TSP8QV", + "security": "confidential" + }, + { + "firstName": "Leroy", + "lastName": "Cole", + "phone": "06 78 56 75 66", + "email": "integer.eu@aol.edu", + "country": "Germany", + "region": "Bengkulu", + "employeeNumber": "SCI84CBR1UF", + "security": "confidential" + }, + { + "firstName": "Axel", + "lastName": "Buckley", + "phone": "09 61 37 41 27", + "email": "dignissim.maecenas@aol.com", + "country": "Germany", + "region": "Luik", + "employeeNumber": "JWL04CDN7BL", + "security": "protected" + }, + { + "firstName": "Martin", + "lastName": "Stuart", + "phone": "09 80 48 88 65", + "email": "feugiat.non@icloud.com", + "country": "Germany", + "region": "Queensland", + "employeeNumber": "LVT07UPM5YB", + "security": "protected" + }, + { + "firstName": "Jerry", + "lastName": "Gonzales", + "phone": "07 24 80 13 06", + "email": "convallis.convallis@aol.org", + "country": "Spain", + "region": "São Paulo", + "employeeNumber": "RBU57EWQ8MI", + "security": "protected" + }, + { + "firstName": "Pandora", + "lastName": "Robinson", + "phone": "05 92 75 54 58", + "email": "libero.mauris@yahoo.couk", + "country": "Spain", + "region": "Meghalaya", + "employeeNumber": "GFM62DCY6SQ", + "security": "top_secret" + }, + { + "firstName": "Xander", + "lastName": "Douglas", + "phone": "08 22 77 36 03", + "email": "arcu.sed@protonmail.couk", + "country": "France", + "region": "Felix", + "employeeNumber": "DIY45MVM4TV", + "security": "top_secret" + }, + { + "firstName": "Tyler", + "lastName": "Webb", + "phone": "07 71 71 93 44", + "email": "pede.praesent@outlook.edu", + "country": "Germany", + "region": "Northern Mindanao", + "employeeNumber": "XWN45SZY1HB", + "security": "top_secret" + }, + { + "firstName": "Martena", + "lastName": "Lynn", + "phone": "06 88 58 74 37", + "email": "magnis.dis@outlook.com", + "country": "Germany", + "region": "Zhōngnán", + "employeeNumber": "MYO35XFT4CC", + "security": "confidential" + }, + { + "firstName": "Clinton", + "lastName": "Bradshaw", + "phone": "08 42 86 17 33", + "email": "venenatis.lacus.etiam@google.net", + "country": "France", + "region": "North West", + "employeeNumber": "MIK59YOF7GO", + "security": "confidential" + }, + { + "firstName": "Giacomo", + "lastName": "House", + "phone": "08 84 84 60 44", + "email": "risus@hotmail.org", + "country": "Germany", + "region": "FATA", + "employeeNumber": "OCQ75JAR6BE", + "security": "confidential" + }, + { + "firstName": "Molly", + "lastName": "Whitehead", + "phone": "06 61 67 75 61", + "email": "interdum.nunc.sollicitudin@yahoo.couk", + "country": "France", + "region": "South Island", + "employeeNumber": "NIS84SJD6FR", + "security": "protected" + }, + { + "firstName": "Luke", + "lastName": "Reed", + "phone": "08 73 28 17 78", + "email": "molestie@protonmail.org", + "country": "Spain", + "region": "Central Region", + "employeeNumber": "KAY46LAI4JN", + "security": "protected" + }, + { + "firstName": "Mason", + "lastName": "Snider", + "phone": "02 57 65 38 41", + "email": "nulla.semper@protonmail.ca", + "country": "Spain", + "region": "Ilocos Region", + "employeeNumber": "YOC20OKT9UN", + "security": "protected" + }, + { + "firstName": "Dieter", + "lastName": "Bright", + "phone": "04 74 61 75 34", + "email": "in.molestie@hotmail.ca", + "country": "Spain", + "region": "Cajamarca", + "employeeNumber": "IJI23SQF8YO", + "security": "top_secret" + }, + { + "firstName": "Tashya", + "lastName": "Vazquez", + "phone": "07 98 70 76 64", + "email": "ultricies@yahoo.edu", + "country": "Spain", + "region": "Zhytomyr oblast", + "employeeNumber": "DBU82FRI2YJ", + "security": "top_secret" + }, + { + "firstName": "Jordan", + "lastName": "Wilder", + "phone": "06 50 84 72 43", + "email": "eget.lacus@aol.ca", + "country": "Spain", + "region": "Central Java", + "employeeNumber": "DGG17XWQ6UM", + "security": "top_secret" + }, + { + "firstName": "Dominique", + "lastName": "Mcfarland", + "phone": "07 17 21 15 05", + "email": "ac@outlook.ca", + "country": "Germany", + "region": "Kursk Oblast", + "employeeNumber": "JIP69EHR6KY", + "security": "confidential" + }, + { + "firstName": "Hyatt", + "lastName": "Marks", + "phone": "03 47 59 28 56", + "email": "at.sem.molestie@yahoo.com", + "country": "France", + "region": "Picardie", + "employeeNumber": "YVU78WCO3LK", + "security": "confidential" + }, + { + "firstName": "Kasper", + "lastName": "Brennan", + "phone": "04 57 71 59 02", + "email": "bibendum@google.ca", + "country": "France", + "region": "Rogaland", + "employeeNumber": "THP01CAA6LJ", + "security": "confidential" + }, + { + "firstName": "Summer", + "lastName": "Crane", + "phone": "05 35 71 18 22", + "email": "ipsum@aol.org", + "country": "Germany", + "region": "North Chungcheong", + "employeeNumber": "UJY85LZO1VG", + "security": "protected" + }, + { + "firstName": "Xenos", + "lastName": "Whitfield", + "phone": "08 30 96 35 54", + "email": "a.feugiat@outlook.couk", + "country": "Spain", + "region": "Kherson oblast", + "employeeNumber": "NVG99IMP6JD", + "security": "protected" + }, + { + "firstName": "Dale", + "lastName": "Lane", + "phone": "06 57 86 21 77", + "email": "lacus@aol.com", + "country": "Spain", + "region": "California", + "employeeNumber": "WOG25MBO3PC", + "security": "protected" + }, + { + "firstName": "Baker", + "lastName": "Knowles", + "phone": "02 59 68 76 19", + "email": "sem.ut@protonmail.edu", + "country": "France", + "region": "Heredia", + "employeeNumber": "JAI31QCC1CS", + "security": "top_secret" + }, + { + "firstName": "Elaine", + "lastName": "Chase", + "phone": "08 96 37 63 57", + "email": "tellus.sem.mollis@icloud.ca", + "country": "France", + "region": "Dalarnas län", + "employeeNumber": "TCF97SQG4US", + "security": "top_secret" + }, + { + "firstName": "Sophia", + "lastName": "Salinas", + "phone": "04 88 58 20 33", + "email": "arcu.sed@outlook.couk", + "country": "Spain", + "region": "East Java", + "employeeNumber": "QPD64DEE4MN", + "security": "top_secret" + }, + { + "firstName": "Ramona", + "lastName": "Sampson", + "phone": "07 66 19 62 82", + "email": "nullam.suscipit@yahoo.ca", + "country": "France", + "region": "Principado de Asturias", + "employeeNumber": "XDX32WIN7KD", + "security": "confidential" + }, + { + "firstName": "Iris", + "lastName": "Berg", + "phone": "01 00 34 62 27", + "email": "ante@aol.ca", + "country": "Spain", + "region": "Atacama", + "employeeNumber": "JOJ64MTG8YN", + "security": "confidential" + }, + { + "firstName": "Calvin", + "lastName": "Joyner", + "phone": "06 84 59 78 28", + "email": "ipsum@google.couk", + "country": "France", + "region": "Lagos", + "employeeNumber": "DPF55GCD6AS", + "security": "confidential" + }, + { + "firstName": "Martina", + "lastName": "Hickman", + "phone": "02 21 24 02 39", + "email": "erat.volutpat@protonmail.com", + "country": "Spain", + "region": "Catalunya", + "employeeNumber": "YRL83GET3VE", + "security": "protected" + }, + { + "firstName": "Xantha", + "lastName": "Montgomery", + "phone": "03 05 74 29 97", + "email": "risus.odio@protonmail.com", + "country": "Germany", + "region": "Merionethshire", + "employeeNumber": "UCQ10RTQ2SI", + "security": "protected" + }, + { + "firstName": "Amy", + "lastName": "Wright", + "phone": "05 65 15 40 63", + "email": "nunc.lectus@aol.com", + "country": "France", + "region": "Penza Oblast", + "employeeNumber": "IPT58WWO5LX", + "security": "protected" + }, + { + "firstName": "Alma", + "lastName": "Schroeder", + "phone": "04 23 46 16 33", + "email": "aliquet.magna.a@protonmail.couk", + "country": "Germany", + "region": "East Kalimantan", + "employeeNumber": "EMH30WSQ3MS", + "security": "top_secret" + }, + { + "firstName": "Marvin", + "lastName": "Bowen", + "phone": "07 42 38 86 27", + "email": "malesuada@aol.couk", + "country": "France", + "region": "Tasmania", + "employeeNumber": "YUQ42QOG7XD", + "security": "top_secret" + }, + { + "firstName": "Ila", + "lastName": "Drake", + "phone": "02 47 45 63 07", + "email": "posuere.enim@google.edu", + "country": "France", + "region": "Gävleborgs län", + "employeeNumber": "RJT61MYB8MY", + "security": "top_secret" + }, + { + "firstName": "Noble", + "lastName": "Cunningham", + "phone": "05 19 20 79 58", + "email": "mollis.non@yahoo.couk", + "country": "Spain", + "region": "Brussels Hoofdstedelijk Gewest", + "employeeNumber": "FCF77JFT8EW", + "security": "confidential" + }, + { + "firstName": "Lilah", + "lastName": "Stewart", + "phone": "05 56 71 95 15", + "email": "tincidunt.orci@google.ca", + "country": "Germany", + "region": "South Australia", + "employeeNumber": "HZI85ZFQ4SH", + "security": "confidential" + }, + { + "firstName": "Gavin", + "lastName": "Bailey", + "phone": "08 25 25 31 93", + "email": "dui.in.sodales@protonmail.net", + "country": "Spain", + "region": "Diyarbakır", + "employeeNumber": "NVH67DKP6FV", + "security": "confidential" + }, + { + "firstName": "Janna", + "lastName": "Hurst", + "phone": "04 26 15 12 98", + "email": "consectetuer.cursus.et@google.couk", + "country": "Spain", + "region": "Ceuta", + "employeeNumber": "EOR61GBW9OL", + "security": "protected" + }, + { + "firstName": "Kylie", + "lastName": "Mullen", + "phone": "02 46 54 90 13", + "email": "lobortis.quam@google.edu", + "country": "Germany", + "region": "Waals-Brabant", + "employeeNumber": "TBS73YUW3PQ", + "security": "protected" + }, + { + "firstName": "MacKensie", + "lastName": "Atkinson", + "phone": "08 03 38 16 24", + "email": "integer.eu@google.org", + "country": "France", + "region": "Sachsen", + "employeeNumber": "NGO64EMM1XG", + "security": "protected" + }, + { + "firstName": "Jack", + "lastName": "Armstrong", + "phone": "07 22 43 06 14", + "email": "varius.nam@protonmail.edu", + "country": "Spain", + "region": "Andaman and Nicobar Islands", + "employeeNumber": "EEI44MCQ4MF", + "security": "top_secret" + }, + { + "firstName": "Karly", + "lastName": "Maxwell", + "phone": "03 93 73 18 84", + "email": "pharetra.nam@icloud.edu", + "country": "Spain", + "region": "Henegouwen", + "employeeNumber": "ABY52MFR3GP", + "security": "top_secret" + }, + { + "firstName": "Lucius", + "lastName": "Baxter", + "phone": "06 37 54 06 88", + "email": "eu.metus@icloud.org", + "country": "France", + "region": "North Region", + "employeeNumber": "UCI44NTO1YV", + "security": "top_secret" + }, + { + "firstName": "Palmer", + "lastName": "Mccall", + "phone": "01 95 80 85 96", + "email": "a.auctor@google.edu", + "country": "Germany", + "region": "Michigan", + "employeeNumber": "QCB22NGM7VN", + "security": "confidential" + }, + { + "firstName": "Reagan", + "lastName": "Lynch", + "phone": "04 15 43 21 21", + "email": "donec.nibh@google.ca", + "country": "France", + "region": "Gävleborgs län", + "employeeNumber": "BUD99RIH9KB", + "security": "confidential" + }, + { + "firstName": "Kibo", + "lastName": "Mcintosh", + "phone": "08 88 35 57 43", + "email": "aliquam@protonmail.net", + "country": "France", + "region": "Małopolskie", + "employeeNumber": "VZR07UEQ2PU", + "security": "confidential" + }, + { + "firstName": "Peter", + "lastName": "Edwards", + "phone": "01 56 71 49 15", + "email": "cras@protonmail.edu", + "country": "Spain", + "region": "Chernivtsi oblast", + "employeeNumber": "FIY93USG7LF", + "security": "protected" + }, + { + "firstName": "Kato", + "lastName": "Parsons", + "phone": "02 15 37 96 48", + "email": "lectus@google.com", + "country": "France", + "region": "Western Australia", + "employeeNumber": "BXB16IOM1OH", + "security": "protected" + }, + { + "firstName": "Suki", + "lastName": "Newman", + "phone": "07 25 82 47 51", + "email": "sed.id@protonmail.ca", + "country": "France", + "region": "Bình Dương", + "employeeNumber": "EFC57JHW4MT", + "security": "protected" + }, + { + "firstName": "Sean", + "lastName": "Tucker", + "phone": "02 55 88 99 01", + "email": "quis.arcu.vel@icloud.ca", + "country": "Germany", + "region": "Kahramanmaraş", + "employeeNumber": "DPF11YBI4FX", + "security": "top_secret" + }, + { + "firstName": "Eve", + "lastName": "Collier", + "phone": "02 38 25 54 78", + "email": "diam.duis.mi@icloud.org", + "country": "Germany", + "region": "Corse", + "employeeNumber": "EXV52JNI9ZG", + "security": "top_secret" + }, + { + "firstName": "Martena", + "lastName": "Grimes", + "phone": "08 78 28 44 11", + "email": "cras.eu@yahoo.edu", + "country": "Germany", + "region": "South Island", + "employeeNumber": "NBX34YIH1OQ", + "security": "top_secret" + }, + { + "firstName": "Eagan", + "lastName": "Foster", + "phone": "05 27 63 48 34", + "email": "donec.sollicitudin@yahoo.couk", + "country": "Germany", + "region": "Munster", + "employeeNumber": "DVD87LBG6UL", + "security": "confidential" + }, + { + "firstName": "Eleanor", + "lastName": "Snow", + "phone": "05 95 41 55 09", + "email": "et.magnis@hotmail.couk", + "country": "Spain", + "region": "Antofagasta", + "employeeNumber": "JDG52IJE1FN", + "security": "confidential" + }, + { + "firstName": "Shaine", + "lastName": "Quinn", + "phone": "06 13 89 12 11", + "email": "sit.amet.lorem@protonmail.edu", + "country": "Germany", + "region": "West Region", + "employeeNumber": "XNW76UKP0DP", + "security": "confidential" + }, + { + "firstName": "Wylie", + "lastName": "Gay", + "phone": "08 99 13 97 15", + "email": "adipiscing.lobortis.risus@hotmail.ca", + "country": "Germany", + "region": "Antofagasta", + "employeeNumber": "OMR67KMM3QR", + "security": "protected" + }, + { + "firstName": "Xenos", + "lastName": "Olson", + "phone": "08 26 78 84 71", + "email": "pede.sagittis@aol.org", + "country": "Spain", + "region": "Paraíba", + "employeeNumber": "EQN94VSX2IJ", + "security": "protected" + }, + { + "firstName": "Beck", + "lastName": "Gray", + "phone": "08 72 82 86 44", + "email": "felis.ullamcorper@google.couk", + "country": "Germany", + "region": "Noord Holland", + "employeeNumber": "VCD88TYG7IX", + "security": "protected" + }, + { + "firstName": "Zeus", + "lastName": "Cherry", + "phone": "03 37 88 63 83", + "email": "dolor.sit@hotmail.edu", + "country": "France", + "region": "Munster", + "employeeNumber": "BTZ36YSA2XP", + "security": "top_secret" + }, + { + "firstName": "Aaron", + "lastName": "Stanton", + "phone": "09 21 34 93 29", + "email": "magna.sed@yahoo.org", + "country": "Germany", + "region": "Mexico City", + "employeeNumber": "UDB51OGB2XO", + "security": "top_secret" + }, + { + "firstName": "Uriah", + "lastName": "Foley", + "phone": "09 39 55 67 05", + "email": "mi.fringilla.mi@yahoo.edu", + "country": "Germany", + "region": "North Gyeongsang", + "employeeNumber": "JXW87GWD3IF", + "security": "top_secret" + }, + { + "firstName": "Ima", + "lastName": "Ewing", + "phone": "02 91 58 54 73", + "email": "vitae@yahoo.com", + "country": "Germany", + "region": "National Capital Region", + "employeeNumber": "JCO37AVA1LH", + "security": "confidential" + } +] diff --git a/examples/cli_demo/findex_db.py b/examples/cli_demo/findex_db.py new file mode 100644 index 0000000..7959928 --- /dev/null +++ b/examples/cli_demo/findex_db.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +import sqlite3 +from cloudproof_py.findex import Findex + +from typing import Dict, List, Set, Tuple, Optional + + +class FindexSQLite(Findex.FindexUpsert, Findex.FindexSearch): + """Implementation of Findex traits for a SQLite backend""" + + def __init__(self, db_conn: sqlite3.Connection) -> None: + super().__init__() + self.conn = db_conn + + def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the entry table + + Args: + entry_uids List[bytes]: uids to query. if None, return the entire table + + Returns: + Dict[bytes, bytes] + """ + str_uids = ",".join("?" * len(entry_uids)) + cur = self.conn.execute( + f"SELECT uid, value FROM entry_table WHERE uid IN ({str_uids})", + entry_uids, + ) + values = cur.fetchall() + output_dict = {} + for value in values: + output_dict[value[0]] = value[1] + return output_dict + + def fetch_all_entry_table_uids(self) -> Set[bytes]: + """Return all UIDs in the Entry Table. + + Returns: + Set[bytes] + """ + cur = self.conn.execute("SELECT uid FROM entry_table") + values = cur.fetchall() + return {value[0] for value in values} + + def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the chain table + + Args: + chain_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes] + """ + str_uids = ",".join("?" * len(chain_uids)) + cur = self.conn.execute( + f"SELECT uid, value FROM chain_table WHERE uid IN ({str_uids})", chain_uids + ) + values = cur.fetchall() + output_dict = {} + for v in values: + output_dict[v[0]] = v[1] + return output_dict + + def upsert_entry_table( + self, entry_updates: Dict[bytes, Tuple[bytes, bytes]] + ) -> Dict[bytes, bytes]: + """Update key-value pairs in the entry table + + Args: + entry_updates (Dict[bytes, Tuple[bytes, bytes]]): uid -> (old_value, new_value) + + Returns: + Dict[bytes, bytes]: entries that failed update (uid -> current value) + """ + rejected_lines: Dict[bytes, bytes] = {} + for uid, (old_val, new_val) in entry_updates.items(): + cursor = self.conn.execute( + """INSERT INTO entry_table(uid,value) VALUES(?,?) + ON CONFLICT (uid) DO UPDATE SET value=? WHERE value=? + """, + (uid, new_val, new_val, old_val), + ) + # Insertion has failed + if cursor.rowcount < 1: + cursor = self.conn.execute( + "SELECT value from entry_table WHERE uid=?", (uid,) + ) + rejected_lines[uid] = cursor.fetchone()[0] + + return rejected_lines + + def insert_entry_table(self, entries_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the entry table + + Args: + entry_items (Dict[bytes, bytes]) + """ + sql_insert_entry = """INSERT INTO entry_table(uid,value) VALUES(?,?)""" + self.conn.executemany( + sql_insert_entry, entries_items.items() + ) # batch insertions + + def insert_chain_table(self, chain_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the chain table + + Args: + chain_items (Dict[bytes, bytes]) + """ + sql_insert_chain = """INSERT INTO chain_table(uid,value) VALUES(?,?)""" + self.conn.executemany(sql_insert_chain, chain_items.items()) # batch insertions + + def remove_entry_table(self, entry_uids: Optional[List[bytes]] = None) -> None: + """Remove entries from entry table + + Args: + entry_uids (List[bytes], optional): uid of entries to delete. + if None, delete all entries + """ + if entry_uids: + self.conn.executemany( + "DELETE FROM entry_table WHERE uid = ?", [(uid,) for uid in entry_uids] + ) + else: + self.conn.execute("DELETE FROM entry_table") + + def remove_chain_table(self, chain_uids: List[bytes]) -> None: + """Remove entries from chain table + + Args: + chain_uids (List[bytes]): uids to remove from the chain table + """ + self.conn.executemany( + "DELETE FROM chain_table WHERE uid = ?", [(uid,) for uid in chain_uids] + ) + + def list_removed_locations(self, locations: List[bytes]) -> List[bytes]: + """Check whether uids still exist in the database + + Args: + db_uids (List[bytes]): uids to check + + Returns: + List[bytes]: list of uids that were removed + """ + res = [] + for uid in locations: + cursor = self.conn.execute("SELECT * FROM users WHERE id = ?", (uid,)) + if not cursor.fetchone(): + res.append(uid) + return res diff --git a/examples/cli_demo/main.py b/examples/cli_demo/main.py new file mode 100644 index 0000000..c4ee924 --- /dev/null +++ b/examples/cli_demo/main.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +import json +import sqlite3 +from secrets import token_bytes +from termcolor import colored + +from findex_db import FindexSQLite +from cloudproof_py import cover_crypt, findex + +sql_create_users_table = """CREATE TABLE IF NOT EXISTS users ( + id BLOB PRIMARY KEY, + firstName BLOB NOT NULL, + lastName BLOB NOT NULL, + email BLOB NOT NULL, + phone BLOB NOT NULL, + country BLOB NOT NULL, + region BLOB NOT NULL, + employeeNumber BLOB NOT NULL, + security BLOB NOT NULL + + );""" + +sql_create_entry_table = """CREATE TABLE IF NOT EXISTS entry_table ( + uid BLOB PRIMARY KEY, + value BLOB NOT NULL + );""" + +sql_create_chain_table = """CREATE TABLE IF NOT EXISTS chain_table ( + uid BLOB PRIMARY KEY, + value BLOB NOT NULL + );""" + + +if __name__ == "__main__": + # Creating DB tables + conn = sqlite3.connect(":memory:") + # Table to store encrypted data + conn.execute(sql_create_users_table) + # Indexing tables required by Findex + conn.execute(sql_create_entry_table) + conn.execute(sql_create_chain_table) + + # Initialize CoverCrypt + policy = cover_crypt.Policy() + policy.add_axis( + cover_crypt.PolicyAxis( + "Country", + ["France", "Spain", "Germany"], + hierarchical=False, + ) + ) + policy.add_axis( + cover_crypt.PolicyAxis("Department", ["MKG", "HR", "SEC"], hierarchical=True) + ) + cc_interface = cover_crypt.CoverCrypt() + cc_master_key, cc_public_key = cc_interface.generate_master_keys(policy) + + # Creating user key with different policy access + key_Alice = cc_interface.generate_user_secret_key( + cc_master_key, + "Country::France && Department::MKG", + policy, + ) + + key_Bob = cc_interface.generate_user_secret_key( + cc_master_key, + "(Country::Spain || Country::Germany) && Department::HR", + policy, + ) + + key_Charlie = cc_interface.generate_user_secret_key( + cc_master_key, + "(Country::France || Country::Spain) && Department::SEC", + policy, + ) + + # Declare encryption scheme + mapping_field_department = { + "firstName": "MKG", + "lastName": "MKG", + "phone": "HR", + "email": "HR", + "country": "HR", + "region": "HR", + "employeeNumber": "SEC", + "security": "SEC", + } + + # Insert user data to DB + Indexing + with open("./data.json", "r", encoding="utf-8") as f: + users = json.load(f) + + # Encryption + Insertion in DB + user_db_uids = [] + for user in users: + # Encrypt each column of the user individually + encrypted_user = [ + cc_interface.encrypt( + policy, + f"Country::{user['country']} && Department::{mapping_field_department[col_name]}", + cc_public_key, + col_value.encode("utf-8"), + ) + for col_name, col_value in user.items() + ] + db_uid = token_bytes(32) + user_db_uids.append(db_uid) + + conn.execute( + """INSERT INTO + users(id,firstName,lastName, email, phone, country, region, employeeNumber, security) + VALUES(?,?,?,?,?,?,?,?,?)""", + (db_uid, *encrypted_user), + ) + print("CoverCrypt: encryption and db insertion done!") + + # Initialize Findex + findex_master_key = findex.MasterKey.random() + findex_label = findex.Label.random() + findex_interface = FindexSQLite(conn) + + # Mapping of the users database UID to the corresponding keywords (firstname, lastname, etc) + mapping_indexed_values_to_keywords = { + findex.IndexedValue.from_location(user_id): [ + keyword.lower() for keyword in user.values() + ] + for user_id, user in zip(user_db_uids, users) + } + # Upsert keywords + findex_interface.upsert( + mapping_indexed_values_to_keywords, findex_master_key, findex_label + ) + print("Findex: Done indexing", len(users), "users") + print( + "Auto completion available: only type the 3 first letters of a word to get results" + ) + activate_auto_completion = input("Activate search auto completion? [y/n] ") == "y" + if activate_auto_completion: + keywords = [keyword.lower() for user in users for keyword in user.values()] + findex_interface.upsert( + findex.utils.generate_auto_completion(keywords), + findex_master_key, + findex_label, + ) + + cc_user_keys = {"Alice": key_Alice, "Bob": key_Bob, "Charlie": key_Charlie} + + while True: + print("\n Available user keys:") + print("\t Alice: Country::France && Department::MKG") + print("\t Bob: (Country::Spain || Country::Germany) && Department::HR") + print("\t Charlie: (Country::France || Country::Spain) && Department::SEC") + + input_user_key = "" + while input_user_key not in cc_user_keys: + input_user_key = input( + "Choose a user key from 'Alice', 'Bob' and 'Charlie': " + ) + user_key = cc_user_keys[input_user_key] + + print("\n You can now search the database for users by providing keywords") + print("Examples of words to try: 'Martin', 'France', 'Kalia'") + keyword = input("Enter a keyword: ").lower() + + # 1. Findex search + found_users = findex_interface.search( + [keyword], findex_master_key, findex_label + ) + if len(found_users) == 0: + print(colored("No user found!", "red", attrs=["bold"])) + continue + found_users_uids = found_users[keyword] + + # 2. Query user database + str_uids = ",".join("?" * len(found_users_uids)) + cur = conn.execute( + f"SELECT * FROM users WHERE id IN ({str_uids})", + found_users_uids, + ) + encrypted_data = cur.fetchall() + + # 3. Decryption + print("Query results:") + for db_user in encrypted_data: + encrypted_user = db_user[1:] # skip the uid + for i, col_name in enumerate(mapping_field_department): + try: + decrypted_value, _ = cc_interface.decrypt( + user_key, encrypted_user[i] + ) + print( + f"{col_name}: {decrypted_value.decode('utf-8'):12.12}", + end=" | ", + ) + except Exception: + # Our user doesn't have access to this data + print( + f"{col_name}:", + colored("Unauthorized", "red", attrs=["bold"]), + end=" | ", + ) + print("") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c252422 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=59"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-data] +cloudproof_py = ["py.typed", "*.pyi"] + +[project] +name = "cloudproof_py" +version = "1.0.0" +authors = [{ name = "Cosmian Tech", email = "tech@cosmian.com" }] +description = "Python library for Cosmian Cloud Proof" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = ["cover-crypt", "findex"] + +[project.urls] +"Homepage" = "https://github.com/Cosmian/cloudproof_python" +"Bug Tracker" = "https://github.com/Cosmian/cloudproof_python/issues" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..53a5397 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +build==0.9.0 +cover_crypt>=8.0.1 +findex>=1.0.1 diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..12647a9 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +install=0 +test=0 +doc=0 + +while getopts "hitd" opt; do + case "$opt" in + h|\?) + echo "Args:" + echo "-i install" + echo "-t run tests" + exit 0 + ;; + i) install=1 + ;; + t) test=1 + ;; + d) doc=1 + ;; + esac +done + +# Build package +python3 -m build +# Optional: automatic install +[ $install -gt 0 ] && pip install --force-reinstall dist/cloudproof_py*.whl +# Optional: run tests +[ $test -gt 0 ] && python3 -m unittest tests/test*.py +# Optional: make doc +[ $doc -gt 0 ] && python3 scripts/extract_lib_types.py && cd docs && make html diff --git a/scripts/extract_lib_types.py b/scripts/extract_lib_types.py new file mode 100644 index 0000000..552e33b --- /dev/null +++ b/scripts/extract_lib_types.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Copy py interface files from PyO3 libs inside our projects +# Theses files are used by sphinx auto-api to generate the docs of the PyO3 libs + +import pkgutil + +if __name__ == "__main__": + SRC_DIR = "src/cloudproof_py" + PKGs_dir = { + "cosmian_cover_crypt": f"{SRC_DIR}/cover_crypt", + "cosmian_findex": f"{SRC_DIR}/findex", + } + + print( + "LOG extract_lib_types: copying function signatures from", ", ".join(PKGs_dir) + ) + + for pkg_name, dest_dir in PKGs_dir.items(): + try: + data = pkgutil.get_data(pkg_name, "__init__.pyi") + if data: + with open(f"{dest_dir}/__init__.docpy", "w", encoding="utf-8") as f: + f.write(f"# file automatically copied from {pkg_name}\n") + f.write(data.decode("utf-8")) + else: + raise FileNotFoundError + except FileNotFoundError: + print(f"WARNING: No typing information found for {pkg_name}") diff --git a/scripts/run_demo.sh b/scripts/run_demo.sh new file mode 100755 index 0000000..6179b6e --- /dev/null +++ b/scripts/run_demo.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eux + +pip install termcolor==2.1.1 types-termcolor==1.1.6 +cd "$(dirname "$0")/../examples/cli_demo" +python3 main.py diff --git a/src/cloudproof_py/__init__.py b/src/cloudproof_py/__init__.py new file mode 100644 index 0000000..500d6f0 --- /dev/null +++ b/src/cloudproof_py/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__all__ = ["findex", "cover_crypt"] diff --git a/src/cloudproof_py/cover_crypt/__init__.py b/src/cloudproof_py/cover_crypt/__init__.py new file mode 100644 index 0000000..16f087e --- /dev/null +++ b/src/cloudproof_py/cover_crypt/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from cosmian_cover_crypt import ( + Attribute, + Policy, + PolicyAxis, + CoverCrypt, + SymmetricKey, + MasterSecretKey, + PublicKey, + UserSecretKey, +) + +__all__ = [ + "Attribute", + "PolicyAxis", + "Policy", + "CoverCrypt", + "SymmetricKey", + "MasterSecretKey", + "PublicKey", + "UserSecretKey", +] diff --git a/src/cloudproof_py/findex/Findex.py b/src/cloudproof_py/findex/Findex.py new file mode 100644 index 0000000..b254687 --- /dev/null +++ b/src/cloudproof_py/findex/Findex.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- +from abc import ABCMeta, abstractmethod +from typing import Dict, List, Optional, Tuple, Set +from cosmian_findex import IndexedValue, Label, MasterKey, InternalFindex + + +class FindexBase(metaclass=ABCMeta): + def __init__(self) -> None: + self.findex_core = InternalFindex() + + +class FindexUpsert(FindexBase, metaclass=ABCMeta): + """Implement this class to use Findex Upsert API""" + + def __init__(self) -> None: + super().__init__() + self.findex_core.set_upsert_callbacks( + self.fetch_entry_table, + self.fetch_chain_table, + self.upsert_entry_table, + self.insert_chain_table, + ) + + def upsert( + self, + indexed_values_and_keywords: Dict[IndexedValue, List[str]], + master_key: MasterKey, + label: Label, + ) -> None: + """Upserts the given relations between `IndexedValue` and `Keyword` into Findex tables. + + Args: + indexed_values_and_keywords (Dict[bytes, List[str]]): map of `IndexedValue` + to a list of `Keyword` + master_key (MasterKey): the user master key + label (Label): label used to allow versioning + """ + self.findex_core.upsert_wrapper(indexed_values_and_keywords, master_key, label) + + @abstractmethod + def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Entry Table. + + Args: + entry_uids (List[bytes], optional): uids to query. if None, return the entire table + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + @abstractmethod + def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Chain Table. + + Args: + chain_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + @abstractmethod + def upsert_entry_table( + self, entry_updates: Dict[bytes, Tuple[bytes, bytes]] + ) -> Dict[bytes, bytes]: + """Update key-value pairs in the Entry Table. + + Args: + entry_updates (Dict[bytes, Tuple[bytes, bytes]]): uid -> (old_value, new_value) + + Returns: + Dict[bytes, bytes]: entries that failed update (uid -> current value) + """ + + @abstractmethod + def insert_chain_table(self, chain_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the Chain Table. + + Args: + chain_items (Dict[bytes, bytes]): uid -> value mapping to insert + """ + + +class FindexSearch(FindexBase, metaclass=ABCMeta): + """Implement this class to use Findex Search API""" + + def __init__(self) -> None: + super().__init__() + self.findex_core.set_search_callbacks( + self.fetch_entry_table, self.fetch_chain_table, self.progress_callback + ) + + @abstractmethod + def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Entry Table. + + Args: + entry_uids (List[bytes], optional): uids to query. if None, return the entire table + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + @abstractmethod + def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Chain Table. + + Args: + chain_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + def progress_callback(self, results: List[IndexedValue]) -> bool: + """Intermediate search results. + + Args: + results (List[IndexedValue]): new locations found + + Returns: + bool: continue recursive search + """ + return True + + def search( + self, + keywords: List[str], + master_key: MasterKey, + label: Label, + max_result_per_keyword: int = 2**32 - 1, + max_depth: int = 100, + ) -> Dict[str, List[bytes]]: + """Recursively search Findex graphs for `Locations` corresponding to the given `Keyword`. + + Args: + keywords (List[str]): keywords to search using Findex + master_key (MasterKey): user secret key + label (Label): public label used in keyword hashing + max_result_per_keyword (int, optional): maximum number of results to fetch per keyword. + max_depth (int, optional): maximum recursion level allowed. Defaults to 100. + + Returns: + Dict[str, List[bytes]]: `Locations` found by `Keyword` + """ + res_indexed_values = self.findex_core.search_wrapper( + keywords, master_key, label, max_result_per_keyword, max_depth + ) + + # return only locations + res_locations: Dict[str, List[bytes]] = {} + for keyword, indexed_values in res_indexed_values.items(): + locations = [] + for indexed_value in indexed_values: + loc = indexed_value.get_location() + if loc: + locations.append(loc) + res_locations[keyword] = locations + + return res_locations + + +class FindexCompact(FindexBase, metaclass=ABCMeta): + """Implement this class to use Findex Compact API""" + + def __init__(self) -> None: + super().__init__() + self.findex_core.set_compact_callbacks( + self.fetch_entry_table, + self.fetch_chain_table, + self.update_lines, + self.list_removed_locations, + self.fetch_all_entry_table_uids, + ) + + @abstractmethod + def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Entry Table. + + Args: + entry_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + @abstractmethod + def fetch_all_entry_table_uids(self) -> Set[bytes]: + """Return all UIDs in the Entry Table. + + Returns: + Set[bytes]: uid set + """ + + @abstractmethod + def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the Chain Table. + + Args: + chain_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes]: uid -> value mapping + """ + + @abstractmethod + def insert_chain_table(self, chain_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the Chain Table. + + Args: + chain_items (Dict[bytes, bytes]): uid -> value mapping to insert + """ + + @abstractmethod + def insert_entry_table(self, entries_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the Entry Table. + + Args: + entries_items (Dict[bytes, bytes]): uid -> value mapping to insert + """ + + @abstractmethod + def remove_entry_table(self, entry_uids: Optional[List[bytes]] = None) -> None: + """Remove entries from Entry Table. + + Args: + entry_uids (List[bytes], optional): uid of entries to delete. if None, + delete all entries + """ + + @abstractmethod + def remove_chain_table(self, chain_uids: List[bytes]) -> None: + """Remove entries from Chain Table. + + Args: + chain_uids (List[bytes]): uids to remove from the chain table + """ + + @abstractmethod + def list_removed_locations(self, locations: List[bytes]) -> List[bytes]: + """Check whether the given `Locations` still exist. + + Args: + locations (List[bytes]): `Locations` to check + + Returns: + List[bytes]: list of `Locations` that were removed + """ + + def update_lines( + self, + removed_chain_table_uids: List[bytes], + new_encrypted_entry_table_items: Dict[bytes, bytes], + new_encrypted_chain_table_items: Dict[bytes, bytes], + ) -> None: + """Example implementation of the compact callback + + Update the database with the new values. + This function should: + + - removes all the Entry Table; + - removes `chain_table_uids_to_remove` from the Chain Table; + - inserts `new_chain_table_items` into the Chain Table; + - inserts `new_entry_table_items` into the Entry Table. + + The order of these operations is not important but has some + implications. This implementation keeps the database small but prevents + using the index during the `update_lines`. + + Override this method if you want another implementation, e.g. : + + 1. saves all Entry Table UIDs; + 2. inserts `new_chain_table_items` into the Chain Table; + 3. inserts `new_entry_table_items` into the Entry Table; + 4. publish new label to users; + 5. remove old lines from the Entry Table (using the saved UIDs in 1.); + 6. removes `chain_table_uids_to_remove` from the Chain Table. + + With this implementation, the index tables are much bigger during a small duration, + but users can continue using the index during the `update_lines`. + """ + + self.remove_entry_table() + self.remove_chain_table(removed_chain_table_uids) + self.insert_chain_table(new_encrypted_chain_table_items) + self.insert_entry_table(new_encrypted_entry_table_items) + + def compact( + self, + num_reindexing_before_full_set: int, + master_key: MasterKey, + new_master_key: MasterKey, + new_label: Label, + ) -> None: + """Performs compacting on the entry and chain tables. + + Args: + num_reindexing_before_full_set (int): number of compacting to do before + being sure that a big portion of the indexes were checked + master_key (MasterKey): current master key + new_master_key (MasterKey): newly generated key + new_label (Label): newly generated label + """ + self.findex_core.compact_wrapper( + num_reindexing_before_full_set, master_key, new_master_key, new_label + ) diff --git a/src/cloudproof_py/findex/__init__.py b/src/cloudproof_py/findex/__init__.py new file mode 100644 index 0000000..5de1d99 --- /dev/null +++ b/src/cloudproof_py/findex/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +from cosmian_findex import IndexedValue, Label, MasterKey +from cloudproof_py.findex import Findex, utils + +__all__ = ["IndexedValue", "Label", "MasterKey", "Findex", "utils"] diff --git a/src/cloudproof_py/findex/utils.py b/src/cloudproof_py/findex/utils.py new file mode 100644 index 0000000..daabffa --- /dev/null +++ b/src/cloudproof_py/findex/utils.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from typing import List, Dict +from cosmian_findex import IndexedValue + + +def generate_auto_completion( + keywords: List[str], + min_word_len: int = 3, + encoding: str = "utf-8", +) -> Dict[IndexedValue, List[str]]: + """Generate a Findex graph of all sub-words from a list of keywords. + For the keyword "Thibaud" with `min_word_len` at 3 it will return + these aliases ["Thi" => "Thib", "Thib" => "Thiba", "Thiba" => "Thibau", "Thibau" => "Thibaud"] + + The original keywords and corresponding locations must be inserted in Findex independently. + + Args: + keywords (List[str]): words to generate sub-words from + min_word_len (int, optional): length of the smallest sub-word to generate. Defaults to 3. + encoding (str, optional): used to encode the string to bytes. Defaults to "utf-8". + + Returns: + Dict[IndexedValue, List[str]]: keyword -> sub-word mapping to upsert in Findex. + """ + res = {} + for word in keywords: + for i in range(min_word_len, len(word)): + res[IndexedValue.from_keyword(word[: i + 1].encode(encoding))] = [word[:i]] + + return res diff --git a/src/cloudproof_py/py.typed b/src/cloudproof_py/py.typed new file mode 100644 index 0000000..1242d43 --- /dev/null +++ b/src/cloudproof_py/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/tests/test_cover_crypt.py b/tests/test_cover_crypt.py new file mode 100644 index 0000000..9745854 --- /dev/null +++ b/tests/test_cover_crypt.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +import unittest +from cloudproof_py.cover_crypt import Policy, PolicyAxis, Attribute, CoverCrypt + + +class TestCoverCrypt(unittest.TestCase): + def test_doc_example(self) -> None: + """ + Creating Policy + """ + + policy = Policy() + policy.add_axis( + PolicyAxis( + "Security Level", + ["Protected", "Confidential", "Top Secret"], + hierarchical=True, + ) + ) + policy.add_axis( + PolicyAxis("Department", ["FIN", "MKG", "HR"], hierarchical=False) + ) + + """ + Generating master keys + """ + + CoverCryptInstance = CoverCrypt() + master_private_key, public_key = CoverCryptInstance.generate_master_keys(policy) + + """ + Messages encryption + """ + + protected_mkg_data = b"protected_mkg_message" + protected_mkg_ciphertext = CoverCryptInstance.encrypt( + policy, + "Department::MKG && Security Level::Protected", + public_key, + protected_mkg_data, + ) + + topSecret_mkg_data = b"top_secret_mkg_message" + topSecret_mkg_ciphertext = CoverCryptInstance.encrypt( + policy, + "Department::MKG && Security Level::Top Secret", + public_key, + topSecret_mkg_data, + ) + + protected_fin_data = b"protected_fin_message" + protected_fin_ciphertext = CoverCryptInstance.encrypt( + policy, + "Department::FIN && Security Level::Protected", + public_key, + protected_fin_data, + ) + + """ + Generating user keys + """ + + confidential_mkg_userKey = CoverCryptInstance.generate_user_secret_key( + master_private_key, + "Department::MKG && Security Level::Confidential", + policy, + ) + + topSecret_mkg_fin_userKey = CoverCryptInstance.generate_user_secret_key( + master_private_key, + "(Department::MKG || Department::FIN) && Security Level::Top Secret", + policy, + ) + + """ + Decryption with the right access policy + """ + + protected_mkg_plaintext, _ = CoverCryptInstance.decrypt( + confidential_mkg_userKey, protected_mkg_ciphertext + ) + self.assertEqual(protected_mkg_plaintext, protected_mkg_data) + + """ + Decryption without the right access will fail + """ + + with self.assertRaises(Exception): + # will throw + CoverCryptInstance.decrypt( + confidential_mkg_userKey, topSecret_mkg_ciphertext + ) + + with self.assertRaises(Exception): + # will throw + CoverCryptInstance.decrypt( + confidential_mkg_userKey, protected_fin_ciphertext + ) + + """ + User with Top Secret access can decrypt messages of all Security Level + within the right Department + """ + + protected_mkg_plaintext2, _ = CoverCryptInstance.decrypt( + topSecret_mkg_fin_userKey, protected_mkg_ciphertext + ) + self.assertEqual(protected_mkg_plaintext2, protected_mkg_data) + + topSecret_mkg_plaintext, _ = CoverCryptInstance.decrypt( + topSecret_mkg_fin_userKey, topSecret_mkg_ciphertext + ) + self.assertEqual(topSecret_mkg_plaintext, topSecret_mkg_data) + + protected_fin_plaintext, _ = CoverCryptInstance.decrypt( + topSecret_mkg_fin_userKey, protected_fin_ciphertext + ) + self.assertEqual(protected_fin_plaintext, protected_fin_data) + + """ + Rotating Attributes + """ + + # make a copy of the current user key + old_confidential_mkg_userKey = confidential_mkg_userKey.deep_copy() + + # rotate MKG attribute + policy.rotate(Attribute("Department", "MKG")) + + # update master keys + CoverCryptInstance.update_master_keys(policy, master_private_key, public_key) + + # update user key + CoverCryptInstance.refresh_user_secret_key( + confidential_mkg_userKey, + "Department::MKG && Security Level::Confidential", + master_private_key, + policy, + keep_old_accesses=True, + ) + + """ + New confidential marketing message + """ + + confidential_mkg_data = b"confidential_secret_mkg_message" + confidential_mkg_ciphertext = CoverCryptInstance.encrypt( + policy, + "Department::MKG && Security Level::Confidential", + public_key, + confidential_mkg_data, + ) + + """ + Decrypting the messages with the rekeyed key + """ + + # decrypting the "old" `protected marketing` message + old_protected_mkg_plaintext, _ = CoverCryptInstance.decrypt( + confidential_mkg_userKey, protected_mkg_ciphertext + ) + self.assertEqual(old_protected_mkg_plaintext, protected_mkg_data) + + # decrypting the "new" `confidential marketing` message + new_confidential_mkg_plaintext, _ = CoverCryptInstance.decrypt( + confidential_mkg_userKey, confidential_mkg_ciphertext + ) + self.assertEqual(new_confidential_mkg_plaintext, confidential_mkg_data) + + # Decrypting the messages with the NON rekeyed key + + # decrypting the "old" `protected marketing` message with the old key works + old_protected_mkg_plaintext, _ = CoverCryptInstance.decrypt( + old_confidential_mkg_userKey, protected_mkg_ciphertext + ) + self.assertEqual(old_protected_mkg_plaintext, protected_mkg_data) + + # decrypting the "new" `confidential marketing` message with the old key fails + with self.assertRaises(Exception): + new_confidential_mkg_plaintext, _ = CoverCryptInstance.decrypt( + old_confidential_mkg_userKey, confidential_mkg_ciphertext + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_findex.py b/tests/test_findex.py new file mode 100644 index 0000000..ee3148b --- /dev/null +++ b/tests/test_findex.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +import sqlite3 +from cloudproof_py.findex import Findex, IndexedValue, MasterKey, Label +from cloudproof_py.findex.utils import generate_auto_completion +from typing import Dict, List, Optional, Set, Tuple +import unittest + + +def create_table(conn, create_table_sql): + try: + conn.execute(create_table_sql) + except sqlite3.Error as e: + print(e) + + +sql_create_users_table = """CREATE TABLE IF NOT EXISTS users ( + id BLOB PRIMARY KEY, + firstName text NOT NULL, + lastName text NOT NULL + );""" + + +sql_create_entry_table = """CREATE TABLE IF NOT EXISTS entry_table ( + uid BLOB PRIMARY KEY, + value BLOB NOT NULL + );""" + +sql_create_chain_table = """CREATE TABLE IF NOT EXISTS chain_table ( + uid BLOB PRIMARY KEY, + value BLOB NOT NULL + );""" + + +class SQLiteBackend(Findex.FindexUpsert, Findex.FindexSearch, Findex.FindexCompact): + # Start implementing Findex methods + + def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the entry table + + Args: + entry_uids (List[bytes]): uids to query. if None, return the entire table + + Returns: + Dict[bytes, bytes] + """ + str_uids = ",".join("?" * len(entry_uids)) + cur = self.conn.execute( + f"SELECT uid, value FROM entry_table WHERE uid IN ({str_uids})", + entry_uids, + ) + values = cur.fetchall() + output_dict = {} + for value in values: + output_dict[value[0]] = value[1] + return output_dict + + def fetch_all_entry_table_uids(self) -> Set[bytes]: + """Return all UIDs in the Entry Table. + + Returns: + Set[bytes] + """ + cur = self.conn.execute("SELECT uid FROM entry_table") + values = cur.fetchall() + + return {value[0] for value in values} + + def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: + """Query the chain table + + Args: + chain_uids (List[bytes]): uids to query + + Returns: + Dict[bytes, bytes] + """ + str_uids = ",".join("?" * len(chain_uids)) + cur = self.conn.execute( + f"SELECT uid, value FROM chain_table WHERE uid IN ({str_uids})", chain_uids + ) + values = cur.fetchall() + output_dict = {} + for v in values: + output_dict[v[0]] = v[1] + return output_dict + + def upsert_entry_table( + self, entry_updates: Dict[bytes, Tuple[bytes, bytes]] + ) -> Dict[bytes, bytes]: + """Update key-value pairs in the entry table + + Args: + entry_updates (Dict[bytes, Tuple[bytes, bytes]]): uid -> (old_value, new_value) + + Returns: + Dict[bytes, bytes]: entries that failed update (uid -> current value) + """ + rejected_lines: Dict[bytes, bytes] = {} + for uid, (old_val, new_val) in entry_updates.items(): + cursor = self.conn.execute( + """INSERT INTO entry_table(uid,value) VALUES(?,?) + ON CONFLICT (uid) DO UPDATE SET value=? WHERE value=? + """, + (uid, new_val, new_val, old_val), + ) + # Insertion has failed + if cursor.rowcount < 1: + cursor = self.conn.execute( + "SELECT value from entry_table WHERE uid=?", (uid,) + ) + rejected_lines[uid] = cursor.fetchone()[0] + return rejected_lines + + def insert_entry_table(self, entries_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the entry table + + Args: + entry_items (Dict[bytes, bytes]) + """ + sql_insert_entry = """INSERT INTO entry_table(uid,value) VALUES(?,?)""" + self.conn.executemany( + sql_insert_entry, entries_items.items() + ) # batch insertions + + def insert_chain_table(self, chain_items: Dict[bytes, bytes]) -> None: + """Insert new key-value pairs in the chain table + + Args: + chain_items (Dict[bytes, bytes]) + """ + sql_insert_chain = """INSERT INTO chain_table(uid,value) VALUES(?,?)""" + self.conn.executemany(sql_insert_chain, chain_items.items()) # batch insertions + + def remove_entry_table(self, entry_uids: Optional[List[bytes]] = None) -> None: + """Remove entries from entry table + + Args: + entry_uids (List[bytes], optional): uid of entries to delete. + if None, delete all entries + """ + if entry_uids: + self.conn.executemany( + "DELETE FROM entry_table WHERE uid = ?", [(uid,) for uid in entry_uids] + ) + else: + self.conn.execute("DELETE FROM entry_table") + + def remove_chain_table(self, chain_uids: List[bytes]) -> None: + """Remove entries from chain table + + Args: + chain_uids (List[bytes]): uids to remove from the chain table + """ + self.conn.executemany( + "DELETE FROM chain_table WHERE uid = ?", [(uid,) for uid in chain_uids] + ) + + def list_removed_locations(self, locations: List[bytes]) -> List[bytes]: + """Check wether uids still exist in the database + + Args: + db_uids (List[bytes]): uids to check + + Returns: + List[bytes]: list of uids that were removed + """ + res = [] + for uid in locations: + cursor = self.conn.execute("SELECT * FROM users WHERE id = ?", (uid,)) + if not cursor.fetchone(): + res.append(uid) + return res + + # End findex trait implementation + + def __init__(self) -> None: + super().__init__() + + # Create database + self.conn = sqlite3.connect(":memory:") + create_table(self.conn, sql_create_users_table) + create_table(self.conn, sql_create_entry_table) + create_table(self.conn, sql_create_chain_table) + + def insert_users(self, new_users: Dict[bytes, List[str]]) -> None: + flat_entries = [(id, *val) for id, val in new_users.items()] + sql_insert_user = """INSERT INTO users(id,firstName,lastName) VALUES(?,?,?)""" + cur = self.conn.cursor() + cur.executemany(sql_insert_user, flat_entries) + + def remove_users(self, users_id: List[bytes]) -> None: + sql_rm_user = """DELETE FROM users WHERE id = ?""" + cur = self.conn.cursor() + cur.executemany(sql_rm_user, [(id,) for id in users_id]) + + def get_num_lines(self, db_table: str) -> int: + return self.conn.execute(f"SELECT COUNT(*) from {db_table};").fetchone()[0] + + +class TestFindexSQLite(unittest.TestCase): + def setUp(self) -> None: + # Init Findex objects + self.db_server = SQLiteBackend() + self.mk = MasterKey.random() + self.label = Label.random() + + self.users = { + b"1": ["Martin", "Sheperd"], + b"2": ["Martial", "Wilkins"], + b"3": ["John", "Sheperd"], + } + self.db_server.insert_users(self.users) + + def test_sqlite_upsert_graph(self) -> None: + # Simple insertion + + indexed_values_and_keywords = { + IndexedValue.from_location(key): value for key, value in self.users.items() + } + self.db_server.upsert(indexed_values_and_keywords, self.mk, self.label) + + res = self.db_server.search(["Sheperd"], self.mk, self.label) + self.assertEqual(len(res["Sheperd"]), 2) + self.assertEqual(self.db_server.get_num_lines("entry_table"), 5) + self.assertEqual(self.db_server.get_num_lines("chain_table"), 5) + + # Generate and upsert graph + + keywords_list = [item for sublist in self.users.values() for item in sublist] + graph = generate_auto_completion(keywords_list) + self.db_server.upsert(graph, self.mk, self.label) + + self.assertEqual(self.db_server.get_num_lines("entry_table"), 18) + self.assertEqual(self.db_server.get_num_lines("chain_table"), 18) + + res = self.db_server.search(["Mar"], self.mk, self.label) + # 2 names starting with Mar + self.assertEqual(len(res["Mar"]), 2) + + res = self.db_server.search(["Mar", "She"], self.mk, self.label) + # all names starting with Mar or She + self.assertEqual(len(res["Mar"]), 2) + self.assertEqual(len(res["She"]), 2) + + def test_sqlite_compact(self) -> None: + indexed_values_and_keywords = { + IndexedValue.from_location(key): value for key, value in self.users.items() + } + self.db_server.upsert(indexed_values_and_keywords, self.mk, self.label) + + # Remove one line in the database before compacting + self.db_server.remove_users([b"1"]) + new_label = Label.random() + new_mk = MasterKey.random() + self.db_server.compact(1, self.mk, new_mk, new_label) + + # only one result left for `Sheperd` + res = self.db_server.search(["Sheperd"], new_mk, new_label) + self.assertEqual(len(res), 1) + + # searching with old label will fail + res = self.db_server.search(["Sheperd"], new_mk, self.label) + self.assertEqual(len(res), 0) + + # searching with old key will fail + res = self.db_server.search(["Sheperd"], self.mk, new_label) + self.assertEqual(len(res), 0) + + +if __name__ == "__main__": + unittest.main()