Skip to content

Commit

Permalink
WIP: add targets directly to RSTUF SQL DB
Browse files Browse the repository at this point in the history 
Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>
  • Loading branch information
Kairo de Araujo committed Jan 23, 2023
1 parent 65a4a35 commit 6a385dd
Show file tree
Hide file tree
Showing 6 changed files with 681 additions and 264 deletions.
4 changes: 3 additions & 1 deletion Pipfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ PyNaCl = "==1.5.0"
requests = "*"
tuf = "==2.0.0"
dynaconf = {extras = ["ini"], version = "*"}
sqlalchemy = "*"
psycopg2 = "*"

[dev-packages]
black = "==22.3.0"
Expand All @@ -37,4 +39,4 @@ build = "*"
pre-commit = "*"

[requires]
python_version = "3.10"
python_version = "3.10"
757 changes: 538 additions & 219 deletions Pipfile.lock
View file

Large diffs are not rendered by default.

89 changes: 89 additions & 0 deletions repository_service_tuf/cli/admin/add_targets.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@

import json

from sqlalchemy import create_engine, Table, MetaData
from datetime import datetime

from repository_service_tuf.cli import click
from repository_service_tuf.cli.admin import admin
from repository_service_tuf.helpers.api_client import (
URL,
Methods,
is_logged,
request_server,
)
from repository_service_tuf.helpers.tuf import Metadata

@admin.command()
@click.option(
"-metadata-url",
help="RSTUF Metadata URL i.e.: http://127.0.0.1 .",
)
@click.option(
"-db-uri",
required=True,
help="RSTUF DB server URI. i.e.: postgresql://postgres:secret@127.0.0.1:5433",
)
@click.option(
"-csv",
required=True,
help="CSV file to import.",
)
@click.option(
"--csv-read-size",
default=2048,
required=False,
show_default=True,
help="CSV read file chunk size.",
)
@click.pass_context
def add_targets(context, metadata_url, db_uri, csv, csv_read_size):
"""
Add targets to RSTUF
"""
settings = context.obj["settings"]
server = settings.get("SERVER")
token = settings.get("TOKEN")
if server and token:
token_access_check = is_logged(server, token)
if token_access_check.state is False:
raise click.ClickException(
f"{str(token_access_check.data)}"
"\n\nTry re-login: 'Repository Service for TUF admin login'"
)
else:
raise click.ClickException("Login first. Run 'rstuf admin login'")

response = request_server(metadata_url, "1.bin.json", Methods.get)
if response.status_code == 404:
print("RSTUF Metadata Targets not found.")

# load all required infrastructure
json_data = json.loads(response.text)
targets = Metadata.from_dict(json_data)
succinct_roles = targets.signed.delegations.succinct_roles
engine = create_engine(f"{db_uri}")
db_metadata = MetaData()
rstuf_targets = Table('rstuf_targets', db_metadata, autoload_with=engine)
connection = engine.connect()
rstuf_db_data = []
with open(csv, 'r') as f:
for line in f:
rstuf_db_data.append(
{
"path": line.split(";")[0],
"info": {
"length": int(line.split(";")[1]),
"hashes": {
line.split(";")[2]: line.split(";")[3]
},
},
"rolename": succinct_roles.get_role_for_target(line.split(";")[0]),
"published": False,
"action": "ADD",
"last_update": datetime.now()

}
)
connection.execute(rstuf_targets.insert(), rstuf_db_data)

71 changes: 37 additions & 34 deletions requirements-dev.txt
View file
Original file line number Diff line number Diff line change
@@ -1,76 +1,79 @@
-i https://pypi.org/simple
alabaster==0.7.12
attrs==22.1.0 ; python_version >= '3.5'
alabaster==0.7.13 ; python_version >= '3.6'
attrs==22.2.0 ; python_version >= '3.6'
babel==2.11.0 ; python_version >= '3.6'
black==22.3.0
build==0.9.0
cachetools==5.2.0 ; python_version ~= '3.7'
build==0.10.0
cachetools==5.2.1 ; python_version ~= '3.7'
certifi==2022.12.7 ; python_version >= '3.6'
cfgv==3.3.1 ; python_full_version >= '3.6.1'
chardet==5.1.0 ; python_version >= '3.7'
charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
charset-normalizer==3.0.1 ; python_full_version >= '3.6.0'
click==8.1.3
colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
coverage==7.0.0
coverage==7.0.5
distlib==0.3.6
docutils==0.17.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
docutils==0.19 ; python_version >= '3.7'
editables==0.3 ; python_version >= '3.1'
exceptiongroup==1.0.4 ; python_version < '3.11'
filelock==3.8.2 ; python_version >= '3.7'
exceptiongroup==1.1.0 ; python_version < '3.11'
filelock==3.9.0 ; python_version >= '3.7'
flake8==6.0.0
hatchling==0.22.0
identify==2.5.10 ; python_version >= '3.7'
identify==2.5.13 ; python_version >= '3.7'
idna==3.4 ; python_version >= '3.5'
imagesize==1.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
iniconfig==1.1.1
isort==5.11.3
iniconfig==2.0.0 ; python_version >= '3.7'
isort==5.11.4
jinja2==3.1.2 ; python_version >= '3.7'
markupsafe==2.1.1 ; python_version >= '3.7'
markupsafe==2.1.2 ; python_version >= '3.7'
mccabe==0.7.0 ; python_version >= '3.6'
mypy==0.991
mypy-extensions==0.4.3
nodeenv==1.7.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
packaging==22.0 ; python_version >= '3.1'
packaging==23.0 ; python_version >= '3.1'
pathspec==0.10.3 ; python_version >= '3.7'
pep517==0.13.0 ; python_version >= '3.6'
pip==22.3.1
platformdirs==2.6.0 ; python_version >= '3.7'
platformdirs==2.6.2 ; python_version >= '3.7'
pluggy==1.0.0 ; python_version >= '3.1'
pre-commit==2.20.0
pre-commit==2.21.0
pretend==1.0.9
pycodestyle==2.10.0 ; python_version >= '3.6'
pyflakes==3.0.1 ; python_version >= '3.6'
pygments==2.13.0 ; python_version >= '3.6'
pyproject-api==1.2.1 ; python_version >= '3.7'
pytest==7.2.0
pytz==2022.7
pygments==2.14.0 ; python_version >= '3.6'
pyproject-api==1.5.0 ; python_version >= '3.7'
pyproject-hooks==1.0.0 ; python_version >= '3.7'
pytest==7.2.1
pytz==2022.7.1
pyyaml==6.0 ; python_version >= '3.6'
requests==2.28.1
setuptools==65.6.3 ; python_version >= '3.7'
requests==2.28.2
setuptools==66.0.0 ; python_version >= '3.7'
snowballstemmer==2.2.0
sphinx==5.3.0
sphinx-rtd-theme==1.1.1
sphinxcontrib-applehelp==1.0.2
sphinx==6.1.3
sphinx-rtd-theme==0.5.1
sphinxcontrib-applehelp==1.0.3
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-plantuml==0.24.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
tomli==2.0.1 ; python_version < '3.11'
tox==4.0.14
tox==4.3.5
typing-extensions==4.4.0 ; python_version >= '3.7'
urllib3==1.26.13 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
urllib3==1.26.14 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
virtualenv==20.17.1 ; python_version >= '3.6'
cffi==1.15.1
commonmark==0.9.1
configobj==5.0.6
configobj==5.0.8
dynaconf[ini]==3.1.11
greenlet==2.0.1 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
markdown-it-py==2.1.0 ; python_version >= '3.7'
mdurl==0.1.2 ; python_version >= '3.7'
psycopg2==2.9.5
pycparser==2.21
pynacl==1.5.0
rich==12.6.0
rich-click==1.6.0
securesystemslib==0.25.0
rich==13.2.0
rich-click==1.6.1
securesystemslib==0.26.0
six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sqlalchemy==1.4.46
tuf==2.0.0
22 changes: 13 additions & 9 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
-i https://pypi.org/simple
certifi==2022.12.7 ; python_version >= '3.6'
cffi==1.15.1
charset-normalizer==2.1.1 ; python_full_version >= '3.6.0'
charset-normalizer==3.0.1 ; python_full_version >= '3.6.0'
click==8.1.3
commonmark==0.9.1
configobj==5.0.6
configobj==5.0.8
dynaconf[ini]==3.1.11
greenlet==2.0.1 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
idna==3.4 ; python_version >= '3.5'
markdown-it-py==2.1.0 ; python_version >= '3.7'
mdurl==0.1.2 ; python_version >= '3.7'
psycopg2==2.9.5
pycparser==2.21
pygments==2.13.0 ; python_version >= '3.6'
pygments==2.14.0 ; python_version >= '3.6'
pynacl==1.5.0
requests==2.28.1
rich==12.6.0
rich-click==1.6.0
securesystemslib==0.25.0
requests==2.28.2
rich==13.2.0
rich-click==1.6.1
securesystemslib==0.26.0
six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sqlalchemy==1.4.46
tuf==2.0.0
urllib3==1.26.13 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
urllib3==1.26.14 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
2 changes: 1 addition & 1 deletion tests/unit/cli/admin/test_ceremony.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ def test_ceremony_start_default_values(
"",
"",
"",
"4",
"14",
"",
]
input_step2 = [
Expand Down

0 comments on commit 6a385dd

Please sign in to comment.