diff --git a/.github/ISSUE_TEMPLATE/user_story.md b/.github/ISSUE_TEMPLATE/user_story.md index 793b33e59..50c14e0aa 100644 --- a/.github/ISSUE_TEMPLATE/user_story.md +++ b/.github/ISSUE_TEMPLATE/user_story.md @@ -9,14 +9,14 @@ assignees: '' #### Describe the user story -List of tasks needed for this user story. Technical description is in the tasks. Additional information, for example a prototype, Figma files, pictures are linked here. +List of tasks needed for this user story. Technical description is in the tasks. Additional information, for example a prototype, Figma files, pictures are linked here. #### Tasks: -- [ ] +- [ ] -#### Testing -Description how the testing will be done. +#### Testing +Description how the testing will be done. #Definition of Done diff --git a/.github/config/.finnishwords.txt b/.github/config/.finnishwords.txt index 932db0909..27c4f8470 100644 --- a/.github/config/.finnishwords.txt +++ b/.github/config/.finnishwords.txt @@ -40,7 +40,7 @@ etsittyä että etusivulle etusivun -Firefoxin +firefoxin hakea haku hakuindeksi @@ -48,7 +48,7 @@ haluat haluatko halutaan halutako -Halutessasi +halutessasi haluttu halutun hinnoittelulaskuri @@ -79,17 +79,10 @@ jakanut jako jakoa jakopyynnöt -jakopyyntöjä jakopyyntöjen +jakopyyntöjä jakotunnus jakotunnusta -jälkeen -jälkeenpäin -jäsenelle -jäseneltä -jäsenet -jäsenille -jäsenten jne jo johtaa @@ -101,6 +94,13 @@ jota julkinen julkiset julkisia +jälkeen +jälkeenpäin +jäsenelle +jäseneltä +jäsenet +jäsenille +jäsenten kaikille kaikki kaikkien @@ -114,33 +114,10 @@ kansiot kansiota kanssa kanssasi -käynnissä -käynnistyy -käytä -käytettävissä -käytetylle -käytössä -käytöstä -käyttää -käyttäen -käyttäjä -käyttäjällä -käyttäjän -käyttäjänimestä -käyttäjänimi -käyttäjätuki -käyttäjätunnuksella -käyttäjätunnustasi -käyttöä -käyttöliittymä -käyttöliittymän -käyttöohje -käyttöoikeudet -käyttörajoista kehittänyt kelpaa -kentällä kenttää +kentällä kerran keskus kestää @@ -179,19 +156,31 @@ kulutus kunnes kuvaus kyseisen +käynnissä +käynnistyy +käytettävissä +käytetylle +käyttäen +käyttäjä +käyttäjällä +käyttäjän +käyttäjänimestä +käyttäjänimi +käyttäjätuki +käyttäjätunnuksella +käyttäjätunnustasi +käyttää +käyttöliittymä +käyttöliittymän +käyttöohje +käyttöoikeudet +käyttörajoista +käyttöä +käytä +käytössä +käytöstä ladata ladattavat -lähetä -lähetetään -lähetettäväksi -lähetettävät -lähetettiin -lähetetyt -lähettäessä -lähettäjän -lähettämistä -lähetys -lähetystä laskettu laskutuksesta laskutusyksiköiden @@ -203,19 +192,14 @@ latauslinkki leikepöydälle liitä linkki -lisää -lisätäksesi +lista lisätietoja lisättiin lisätty lisättyjä -lista +lisätäksesi +lisää loppuminen -löydetty -löydy -löytää -löytävät -löytynyt luettelo luku lukuoikeus @@ -224,8 +208,22 @@ luonnin luonti luotu lupa -määrä -määrästä +lähetettiin +lähetettäväksi +lähetettävät +lähetetyt +lähetetään +lähettäessä +lähettäjän +lähettämistä +lähetys +lähetystä +lähetä +löydetty +löydy +löytynyt +löytävät +löytää mahdollisia mahdollista megatavun @@ -249,23 +247,25 @@ muutamia muutettiin muuttaa muuttaaksesi -näkyvissä -nämä +määrä +määrästä nappi nappia navigointia -näytä -näytetään -näytetyistä -näyttäminen niiden -nimeä nimellä nimetä +nimeä nimi nykyinen nykyisen nykyisiä +näkyvissä +nämä +näytetyistä +näytetään +näyttäminen +näytä objekteja objekti objektia @@ -304,12 +304,9 @@ ota ottaneesi ovat oy -päänäkymään -pääsy paina painikkeella painikkeesta -päivitä pakollista palaa palautus @@ -358,10 +355,13 @@ projektitunnuksen projektitunnukset pudota puuttuvasta -pyydä pyydetty pyydetyn +pyydä pyyntö +päivitä +päänäkymään +pääsy rajan rajattu rajatun @@ -375,14 +375,6 @@ resurssienkäyttö resurssit rivinvaihtoa saavutettavuus -sähköpostitse -säiliö -säiliöitä -säiliön -säiliöön -säiliöt -säiliötä -säilön salaa salaaminen salasana @@ -412,10 +404,10 @@ seuraava siirry siirto sinulle -sisään -sisäänkirjauksen sisällön sisältö +sisään +sisäänkirjauksen sivu sivua sivulla @@ -437,19 +429,20 @@ synkronoi synkronoitavaksi synkronoitiin syötä -tägejä -tägillä -tägit -tähän +sähköpostitse +säiliö +säiliöitä +säiliön +säiliöt +säiliötä +säiliöön +säilön tai takaisin -tällä tallenna tallennettujen tallennustilaan talteen -tämä -tämän tapauksessa tarkastella tarkat @@ -457,11 +450,8 @@ tarkistathan tarkistussumma tarkoittaen tarvitset -tässä -tätä taustalla tavallisesti -täytyy tekijänoikeustiedot tekstinä tiedosto @@ -506,10 +496,20 @@ tunnus tunti tuo tuottaa -tyhjä tyhjennä -työkaluvinkki +tyhjä tyyppi +työkaluvinkki +tägejä +tägillä +tägit +tähän +tällä +tämä +tämän +tässä +tätä +täytyy ulos useille useisiin @@ -520,10 +520,8 @@ uuden uusi uusia vaatii -vähintään vaihda vaihto -väliaikaiseen valikkoon valikossa valinnan @@ -551,11 +549,13 @@ voidaksesi voit voivat vuoksi +vähintään +väliaikaiseen yhdelle yhden -yhtään yhteenlaskettu yhteys +yhtään yksityinen yksityisen yksityisestä diff --git a/.github/config/.wordlist.txt b/.github/config/.wordlist.txt index 13eaa683b..0f4586b7f 100644 --- a/.github/config/.wordlist.txt +++ b/.github/config/.wordlist.txt @@ -21,9 +21,10 @@ cp cryptfiles csc cscfi +dev devel devenv -Dockerfile +dockerfile dockerfiles drag'n'drop dropdown @@ -69,14 +70,16 @@ plaintext png pnpm postgres +postgresql pouta proxying py quickstart +redis saml serviceworker sha -sLo +slo ssl sso subcommands @@ -93,7 +96,7 @@ tooltip tox ttl txt -Ue +ue ui untrusted url @@ -105,4 +108,5 @@ vue wasm webdrivers websso +whitelisted yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 000000000..611993d60 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,15 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - run: pip install mypy .[dev] + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index b32139b50..bf03bc3b2 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -17,4 +17,4 @@ jobs: - uses: rojopolis/spellcheck-github-actions@0.30.0 name: Spellcheck with: - config_path: .github/config/.spellcheck.yml + config_path: .pyspelling.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..27eaf85bb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,83 @@ +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: no-commit-to-branch + args: [--branch, devel, --branch, master] + - id: check-toml + - id: check-yaml + exclude: swift_browser_ui_frontend/pnpm-lock.yaml + - id: check-ast + - id: check-docstring-first + - id: check-case-conflict + - id: check-merge-conflict + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.259 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + files: ^swift_browser_ui/ + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + args: [-l, "90"] + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.5 + hooks: + - id: bandit + files: ^swift_browser_ui/ + + - repo: local + hooks: + - id: eslint + name: Lint frontend + entry: ./scripts/eslint.sh + args: [--prefix, swift_browser_ui_frontend, run, lint] + files: ^swift_browser_ui_frontend + language: system + pass_filenames: false + + - id: mypy + name: mypy + entry: mypy + language: system + types: [python] + files: ^swift_browser_ui/ + args: [--ignore-missing-imports, --no-namespace-packages] + require_serial: true + + - id: sort + name: sort dictionaries + entry: ./scripts/sort.sh + language: script + files: ^.github/config/.*.txt + require_serial: true + pass_filenames: false + + - id: pyspelling-docs + name: spellcheck markdown and documentation + entry: ./scripts/pyspelling.sh + language: script + args: [markdown] + types_or: [markdown, rst] + require_serial: true + + - id: pyspelling-frontend + name: spellcheck frontend strings + entry: ./scripts/pyspelling.sh + language: script + args: [frontend] + types: [javascript] + files: swift_browser_ui_frontend/src/common/lang.js + require_serial: true diff --git a/.github/config/.spellcheck.yml b/.pyspelling.yml similarity index 89% rename from .github/config/.spellcheck.yml rename to .pyspelling.yml index 085e065cc..0ea234814 100644 --- a/.github/config/.spellcheck.yml +++ b/.pyspelling.yml @@ -1,5 +1,5 @@ matrix: -- name: Markdown +- name: markdown aspell: lang: en ignore-case: true @@ -13,7 +13,7 @@ matrix: context_visible_first: true escapes: '\\[\\`~]' delimiters: - # Ignore text between inline back ticks as this is code or hightlight words + # Ignore text between inline back ticks as this is code or highlight words - open: '(?P`+)' close: '(?P=open)' # Ignore surrounded in <> as in RST it is link @@ -23,7 +23,7 @@ matrix: - 'docs/source/*.rst' - '**/*.md' default_encoding: utf-8 -- name: English Language spellcheck +- name: frontend aspell: lang: en ignore-case: true @@ -39,4 +39,4 @@ matrix: strings: true decode_escapes: false sources: - - swift_browser_ui_frontend/src/common/lang.js \ No newline at end of file + - swift_browser_ui_frontend/src/common/lang.js diff --git a/README.md b/README.md index ec0a1d048..755e5f55c 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ The current frontend can be found at: `127.0.0.1:8080`. ### Development swift-browser-ui is composed of 4 components: `request`, `sharing`, `ui`, and `upload`. All of them must be run to have access to all features. -They depend on a Redis instance for session cache, Postgres database for the sharing and +They depend on a Redis instance for session cache, Postgres database for the sharing and request functionality, and the object storage backend. You will also need docker with Buildkit to build the keystone-swift docker image. @@ -118,6 +118,7 @@ Install python dependencies, optionally in a virtual environment. python3 -m venv venv --prompt swiftui # Optional step, creates python virtual environment source venv/bin/activate # activates virtual environment pip install -Ue .[docs,test,dev] +pre-commit install ``` Set up the environment variables @@ -154,7 +155,7 @@ This guide assumes you're using `devenv` as the domain name. Replace this with the domain you're certificate sings, and if necessary, add it to `/etc/hosts` so it's resolvable both in docker, and locally. -Additional setup is required in your environment file. You'll need to +Additional setup is required in your environment file. You'll need to configure the following keys to point to whatever hostname will be used to access the service. Additionally you should allow all hosts, assuming your machine is in a secure network when developing. In case you trust @@ -293,7 +294,7 @@ After following the development steps above, `cypress` should already be install cd swift_browser_ui_frontend You can run the tests in headless mode - + npx cypress run Or you can use the interactive version diff --git a/devproxy/nginx.conf b/devproxy/nginx.conf index b12106fb4..25b16f2f5 100644 --- a/devproxy/nginx.conf +++ b/devproxy/nginx.conf @@ -23,7 +23,7 @@ http { } client_max_body_size 0; - + server { listen 9443 ssl; ssl_certificate /etc/ssl/swiftui-proxy.crt; diff --git a/docs/_static/api.yml b/docs/_static/api.yml index d95e08f32..107be6162 100644 --- a/docs/_static/api.yml +++ b/docs/_static/api.yml @@ -6,15 +6,15 @@ info: license: name: MIT tags: - - name: Frontend + - name: Frontend description: Endpoints for sharing request "Filtered proxied API from Openstack Swift used for a browser application" - - name: Sharing Request + - name: Sharing Request description: Endpoints for sharing request "Database and API for storing / querying Openstack Swift ACL share action request information." - - name: Sharing Account + - name: Sharing Account description: Endpoints for sharing request "Database and API for storing / querying Openstack Swift ACL information" - - name: Upload/Download + - name: Upload/Download description: Endpoints for uploading and downloading containers/objects - + paths: /health: get: @@ -33,7 +33,7 @@ paths: /request/user/{user}/{container}: post: tags: - - Sharing Request + - Sharing Request summary: Publish a new request for access to a container parameters: - name: user @@ -65,7 +65,7 @@ paths: $ref: "#/components/schemas/NewRequest" delete: tags: - - Sharing Request + - Sharing Request summary: Delete an existing request for access parameters: - name: user @@ -98,7 +98,7 @@ paths: example: OK options: tags: - - Sharing Request + - Sharing Request summary: Handle preflight for existing request for access parameters: - name: user @@ -136,7 +136,7 @@ paths: /request/user/{user}: get: tags: - - Sharing Request + - Sharing Request summary: List container share requests made by a user. parameters: - name: user @@ -156,7 +156,7 @@ paths: /request/owner/{user}: get: tags: - - Sharing Request + - Sharing Request summary: List the requests owned by the user (made for the user). parameters: - name: user @@ -176,7 +176,7 @@ paths: /request/container/{container}: get: tags: - - Sharing Request + - Sharing Request summary: List share requests made for a container. parameters: - name: container @@ -193,11 +193,11 @@ paths: application/json: schema: $ref: "#/components/schemas/ContainerRequests" - + /access/{user}: get: tags: - - Sharing Account + - Sharing Account summary: List container names that have been shared to the user. parameters: - name: user @@ -219,7 +219,7 @@ paths: /access/{user}/{container}: get: tags: - - Sharing Account + - Sharing Account summary: Get details of a container that has been shared to the user. parameters: - name: container @@ -255,7 +255,7 @@ paths: /share/{owner}: get: tags: - - Sharing Account + - Sharing Account summary: List container names that the user has shared to someone else. parameters: - name: owner @@ -277,7 +277,7 @@ paths: /share/{owner}/{container}: get: tags: - - Sharing Account + - Sharing Account summary: Get details of a container that the user has shared. parameters: - name: container @@ -305,7 +305,7 @@ paths: description: Client Error post: tags: - - Sharing Account + - Sharing Account summary: Give user / users access rights on a specified container. parameters: - name: container @@ -354,7 +354,7 @@ paths: description: Client Error patch: tags: - - Sharing Account + - Sharing Account summary: Edit specified user's access rights on a specified container. parameters: - name: container @@ -396,7 +396,7 @@ paths: description: Client Error delete: tags: - - Sharing Account + - Sharing Account summary: Revoke specified access to container from the user/users specified in the query. parameters: - name: container @@ -429,7 +429,7 @@ paths: description: Client Error options: tags: - - Sharing Account + - Sharing Account summary: Serve correct response headers to allowed DELETE preflight query. parameters: - name: container @@ -465,7 +465,7 @@ paths: schema: type: string example: 84600 - + /token/{project}/{id}: post: tags: @@ -572,7 +572,7 @@ paths: /api/username: get: tags: - - Frontend + - Frontend summary: Get Openstack username for the authenticated user. responses: 200: @@ -586,7 +586,7 @@ paths: /api/projects: get: tags: - - Frontend + - Frontend summary: Get Openstack projects available for the authenticated user. responses: 200: @@ -600,7 +600,7 @@ paths: /api/{project}/acl: get: tags: - - Frontend + - Frontend summary: Fetch a compilation of ACL information for sharing discovery. parameters: - name: project @@ -622,7 +622,7 @@ paths: /api/{project}/address: get: tags: - - Frontend + - Frontend summary: Get the project specific object storage address. parameters: - name: project @@ -645,7 +645,7 @@ paths: /api/meta/{project}: get: tags: - - Frontend + - Frontend summary: Get the filtered project metadata for the currently active project. parameters: - name: project @@ -669,7 +669,7 @@ paths: /api/{project}: get: tags: - - Frontend + - Frontend summary: Get the listing for containers for specified project parameters: - name: project @@ -702,7 +702,7 @@ paths: /api/{project}/{container}: get: tags: - - Frontend + - Frontend summary: Get the listing for objects in a given container. parameters: - name: project @@ -815,7 +815,7 @@ paths: /api/meta/{project}/{container}: get: tags: - - Frontend + - Frontend summary: Get container metadata. parameters: - name: project @@ -902,7 +902,7 @@ paths: /api/{project}/{container}/{object}: get: tags: - - Frontend + - Frontend summary: Download the object specified in the download link. parameters: - name: project @@ -1153,7 +1153,7 @@ paths: schema: type: string format: binary - + /{project}/{container}: post: tags: @@ -1289,7 +1289,7 @@ components: type: array items: $ref: "#/components/schemas/ListedRequest" - + ListedContainer: type: object properties: @@ -1344,7 +1344,7 @@ components: type: array items: $ref: "#/components/schemas/DetailedContainer" - + Username: type: string example: test_user_name @@ -1386,24 +1386,24 @@ components: type: integer example: 4294967296 description: The total amount of storage usage in bytes for the currently active project. - + ProjectAcl: type: object properties: - address: + address: type: string example: host access: type: object example: - containerName1: - ownerID1: + containerName1: + ownerID1: read: "*" write: "*" containerName2: - ownerID2: + ownerID2: read: "*" - + Container: type: object properties: @@ -1483,7 +1483,7 @@ components: type: array items: $ref: '#/components/schemas/MetaItem' - + Health: type: object required: @@ -1523,4 +1523,3 @@ components: items: type: string example: database - diff --git a/docs/source/_static/style.css b/docs/source/_static/style.css index 674649fc8..aba37f9bd 100644 --- a/docs/source/_static/style.css +++ b/docs/source/_static/style.css @@ -8,9 +8,8 @@ this as on RTD they are loaded after this stylesheet */ white-space: normal !important; } - + .wy-table-responsive { overflow: visible !important; } } - diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 77718c6c3..84d28fb2b 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -35,7 +35,7 @@ for the database) Sharing functionality back-end ------------------------------ Sharing functionality should be run by running it in a container. Easiest -way to do this is to use the docker-compose fields provided in the +way to do this is to use the docker-compose fields provided in the `deployment example repository. `_ The sharing functionality requires the following environment variables to be present in order to work: @@ -87,7 +87,7 @@ to be present in order to work: Upload runner back-end ---------------------- SwiftUI upload runner should be run by running it in a container. Easiest way -to do this is to use the docker-compose files provided in the +to do this is to use the docker-compose files provided in the `deployment example repository. `_ The upload runner requires the following environment variables to be present in order to work: diff --git a/docs/source/instructions.rst b/docs/source/instructions.rst index e5c8885ea..9d3aede8c 100644 --- a/docs/source/instructions.rst +++ b/docs/source/instructions.rst @@ -19,7 +19,7 @@ The program can be installed with pip from the git repository: can be found from the git repositories. The instructions for getting the services up and running can be found in their respective repositories, and partly under the *Deployment* section. - + * https://github.com/cscfi/swift-x-account-sharing * https://github.com/cscfi/swift-sharing-request * https://github.com/cscfi/swiftui-upload-runner @@ -82,7 +82,7 @@ For the Pouta test environment with NGINX TLS termination proxy in use:: For the Pouta production environment for testing unsecurely without trust:: export BROWSER_START_AUTH_ENDPOINT_URL="https://pouta.csc.fi:5001/v3" - + Setting up TLS termination proxy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/tech.rst b/docs/source/tech.rst index 0104b0886..3958d5011 100644 --- a/docs/source/tech.rst +++ b/docs/source/tech.rst @@ -56,4 +56,3 @@ the API during normal usage. The api is documented in the api.yml file, that conforms to the OpenAPI specification (the file can be rendered with the `swagger editor `_): - diff --git a/docs/source/testing.rst b/docs/source/testing.rst index 7bc7e14e9..f52d630e4 100644 --- a/docs/source/testing.rst +++ b/docs/source/testing.rst @@ -39,7 +39,7 @@ and the tests are developed for both Firefox and Chrome web browsers. $ cd swift_browser_ui_frontend/ $ npm install -g pnpm@7 - $ pnpm install + $ pnpm install $ pnpm run build $ cd .. $ pnpm install cypress diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 82a41bfcb..4b7bb68a0 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -2,7 +2,7 @@ Getting started =============== .. note:: - Please note that the things related to project development aren’t + Please note that the things related to project development aren't documented here, and everything on this page is only related to the running of the program diff --git a/pyproject.toml b/pyproject.toml index ef8b2dcff..fff8ccb28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ test = [ "coverage==7.2.1", "flake8-docstrings==1.7.0", "flake8==6.0.0", - "mypy==1.0.1", + "mypy==1.1.1", "pytest-cov==4.0.0", "pytest-xdist==3.2.0", "pytest==7.2.1", @@ -57,7 +57,12 @@ ui_test = [ ] dev = [ "honcho==1.1.0", + "pre-commit==3.2.1", "pyspelling==2.8.2", + "ruff==0.0.259", + "types-certifi", + "types-redis", + "types-requests", ] [project.scripts] @@ -72,3 +77,41 @@ Documentation = "https://swift-browser-ui.readthedocs.io" [tool.hatch.version] path = "swift_browser_ui/__init__.py" + +[tool.black] +line-length = 90 +target-version = ['py310'] + +[tool.isort] +atomic = true +profile = "black" +line_length = 90 +py_version=310 + +[tool.ruff] +line-length = 90 +target-version = "py310" + +# https://beta.ruff.rs/docs/rules/ +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + # "ANN", # flake8-annotations + "C", # flake8-comprehensions + "B", # flake8-bugbear + "D", # pydocstyle +# "UP", # pyupgrade + "S", # Bandit +] + +ignore = [ + "ANN101", # Missing type annotation for `self` in method + "E501", # line too long, handled by black + "B904", # do not perform function calls in argument defaults + "PLR2004", # magic value used in comparison + "S113", # Probable use of requests call without timeout + "D203", # one-blank-line-before-class + "D213", # multi-line-summary-second-line +] diff --git a/scripts/eslint.sh b/scripts/eslint.sh new file mode 100755 index 000000000..ed3700cf0 --- /dev/null +++ b/scripts/eslint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# eslint for pre-commit. Skips run if `pnpm` or eslint are not installed + +if ! command -v pnpm > /dev/null 2>&1; then + echo "pnpm not installed, not running eslint as pre-commit hook" + exit 0 +fi + +if ! test -f swift_browser_ui_frontend/node_modules/.bin/eslint; then + echo "eslint is not installed, not running eslint as pre-commit hook" + exit 0 +fi + +pnpm --prefix swift_browser_ui_frontend run lint diff --git a/scripts/pyspelling.sh b/scripts/pyspelling.sh new file mode 100755 index 000000000..e74c3b54f --- /dev/null +++ b/scripts/pyspelling.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +if ! command -v pyspelling > /dev/null 2>&1; then + echo "pyspelling not installed, not running as pre-commit hook" + exit 0 +elif ! aspell -v > /dev/null 2>&1; then + echo "aspell is not installed, not running as pre-commit hook" + exit 0 +fi + +check_files="" +for arg in "${@:2:$#}" +do + check_files+="--source $arg " +done + +pyspelling -v --name $1 ${check_files} diff --git a/scripts/sort.sh b/scripts/sort.sh new file mode 100755 index 000000000..3dc782512 --- /dev/null +++ b/scripts/sort.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Sorts dictionary, converts to lower case, and remove duplicates + +# Makes sorting consistent across machines +export LC_ALL=C + +for file in ".github/config/.wordlist.txt" ".github/config/.finnishwords.txt"; do + sort "${file}" | tr "[:upper:]" "[:lower:]" | sort -u -o "${file}" +done diff --git a/swift_browser_ui/__init__.py b/swift_browser_ui/__init__.py index add95aa80..398774b1e 100644 --- a/swift_browser_ui/__init__.py +++ b/swift_browser_ui/__init__.py @@ -1,5 +1,4 @@ -""" -Web application for browsing OS Swift object storage. +"""Web application for browsing OS Swift object storage. Uses OS keystone for authentication and python-swiftclient for communicating with the object storage. diff --git a/swift_browser_ui/common/common_handlers.py b/swift_browser_ui/common/common_handlers.py index a8a0fc8bb..260019d5f 100644 --- a/swift_browser_ui/common/common_handlers.py +++ b/swift_browser_ui/common/common_handlers.py @@ -1,9 +1,10 @@ """Module containing handlers common to swift_browser_ui services.""" -import aiohttp.web import typing +import aiohttp.web + async def handle_delete_preflight( _: typing.Union[aiohttp.web.Request, None] diff --git a/swift_browser_ui/common/common_middleware.py b/swift_browser_ui/common/common_middleware.py index 5d8ac71a9..5ffd6da78 100644 --- a/swift_browser_ui/common/common_middleware.py +++ b/swift_browser_ui/common/common_middleware.py @@ -8,9 +8,8 @@ import aiohttp.web import asyncpg.exceptions -import swift_browser_ui.common.types import swift_browser_ui.common.signature - +import swift_browser_ui.common.types LOGGER = logging.getLogger("swift_browser_ui.common.common_middleware") LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/common/common_util.py b/swift_browser_ui/common/common_util.py index f9f92603f..fe4bc8ffb 100644 --- a/swift_browser_ui/common/common_util.py +++ b/swift_browser_ui/common/common_util.py @@ -1,7 +1,9 @@ """Module for swift_browser_ui common utility functions.""" +import asyncio import os +import random import aiohttp.web @@ -12,3 +14,8 @@ async def read_in_keys(app: aiohttp.web.Application) -> None: app["tokens"] = keys.split(",") if keys is not None else [] if app["tokens"]: app["tokens"] = [token.encode("utf-8") for token in app["tokens"]] + + +async def sleep_random(): + """Sleep a random time.""" + return await asyncio.sleep(random.randint(2, 5)) # nosec # noqa: S311 diff --git a/swift_browser_ui/common/signature.py b/swift_browser_ui/common/signature.py index 1fb553423..86cba070f 100644 --- a/swift_browser_ui/common/signature.py +++ b/swift_browser_ui/common/signature.py @@ -2,15 +2,14 @@ import hmac -import time -import os import logging -import typing +import os import secrets +import time +import typing import aiohttp.web - LOGGER = logging.getLogger("swift_browser_ui.common.signature") LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/common/types.py b/swift_browser_ui/common/types.py index 504d9b02e..461b6830d 100644 --- a/swift_browser_ui/common/types.py +++ b/swift_browser_ui/common/types.py @@ -5,7 +5,6 @@ import aiohttp.web - AiohttpHandler = typing.Callable[ [aiohttp.web.Request], typing.Coroutine[typing.Awaitable, typing.Any, aiohttp.web.Response], diff --git a/swift_browser_ui/common/vault_client.py b/swift_browser_ui/common/vault_client.py index 5cb1096ba..a4f0f626c 100644 --- a/swift_browser_ui/common/vault_client.py +++ b/swift_browser_ui/common/vault_client.py @@ -1,4 +1,4 @@ -"""Vault c4ghtransit client""" +"""Vault c4ghtransit client.""" import asyncio import logging import os @@ -262,7 +262,7 @@ async def get_key() -> str: async def put_whitelist_key( self, project: str, flavor: str, public_key: bytes ) -> None: - """Updates the project's whitelisted key. + """Update the project's whitelisted key. :param project: Project ID :param flavor: Public key flavor: one of crypt4gh or ed25519 @@ -278,7 +278,7 @@ async def put_whitelist_key( ) async def remove_whitelist_key(self, project: str) -> None: - """Deletes the project's whitelisted key. + """Delete the project's whitelisted key. :param project: Project ID """ @@ -287,6 +287,12 @@ async def remove_whitelist_key(self, project: str) -> None: ) async def get_header(self, project: str, container: str, path: str) -> str: + """Retrieve header. + + :param project: Project ID + :param container: container name + :param path: object path + """ header_response = await self._request( "GET", f"c4ghtransit/files/{project}/{container}/{path}", @@ -299,6 +305,13 @@ async def get_header(self, project: str, container: str, path: str) -> str: async def put_header( self, project: str, container: str, path: str, header: str ) -> None: + """Update header. + + :param project: Project ID + :param container: container name + :param path: object path + :param header: header as b64 encoded string + """ await self._request( "POST", f"c4ghtransit/files/{project}/{container}/{path}", diff --git a/swift_browser_ui/launcher.py b/swift_browser_ui/launcher.py index b656143e6..6d87c38da 100644 --- a/swift_browser_ui/launcher.py +++ b/swift_browser_ui/launcher.py @@ -1,8 +1,8 @@ """Module containing launchers for the different services.""" -import swift_browser_ui.ui.shell -import swift_browser_ui.sharing.server import swift_browser_ui.request.server +import swift_browser_ui.sharing.server +import swift_browser_ui.ui.shell import swift_browser_ui.upload.server diff --git a/swift_browser_ui/request/api.py b/swift_browser_ui/request/api.py index aaaddf6e0..b91801f65 100644 --- a/swift_browser_ui/request/api.py +++ b/swift_browser_ui/request/api.py @@ -3,8 +3,8 @@ import logging import os -import aiohttp.web +import aiohttp.web MODULE_LOGGER = logging.getLogger("api") MODULE_LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/request/bindings/__init__.py b/swift_browser_ui/request/bindings/__init__.py index e69de29bb..b3f58f027 100644 --- a/swift_browser_ui/request/bindings/__init__.py +++ b/swift_browser_ui/request/bindings/__init__.py @@ -0,0 +1 @@ +"""Async Python bindings for the swift-x-account-sharing backend.""" diff --git a/swift_browser_ui/request/bindings/bind.py b/swift_browser_ui/request/bindings/bind.py index cd9723552..e279d2347 100644 --- a/swift_browser_ui/request/bindings/bind.py +++ b/swift_browser_ui/request/bindings/bind.py @@ -1,17 +1,16 @@ """Async Python bindings for the swift-x-account-sharing backend.""" -import os import json -import typing import logging -import aiohttp - -import swift_browser_ui.common.signature - +import os import ssl +import typing + +import aiohttp import certifi +import swift_browser_ui.common.signature ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) diff --git a/swift_browser_ui/request/db.py b/swift_browser_ui/request/db.py index 1b8dac1eb..e1dfd180d 100644 --- a/swift_browser_ui/request/db.py +++ b/swift_browser_ui/request/db.py @@ -4,11 +4,10 @@ import logging import os import typing -import asyncio -import random import asyncpg +from swift_browser_ui.common.common_util import sleep_random MODULE_LOGGER = logging.getLogger("db") MODULE_LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) @@ -46,13 +45,13 @@ async def open(self) -> None: "Failed to establish database connection. " "Pool will retry reconnection automatically...", ) - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() except asyncpg.exceptions.InvalidPasswordError: self.log.error("Invalid username or password for database.") - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() except asyncpg.exceptions.CannotConnectNowError: self.log.error("Database is not ready yet.") - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() async def close(self) -> None: """Gracefully close the database.""" diff --git a/swift_browser_ui/request/server.py b/swift_browser_ui/request/server.py index cf300a8ce..14f639597 100644 --- a/swift_browser_ui/request/server.py +++ b/swift_browser_ui/request/server.py @@ -1,32 +1,30 @@ """Share request backend module.""" -import sys -import logging import asyncio +import logging +import sys +import typing import aiohttp.web import uvloop -import typing -import swift_browser_ui.common.common_middleware import swift_browser_ui.common.common_handlers +import swift_browser_ui.common.common_middleware import swift_browser_ui.common.common_util - from swift_browser_ui.request.api import ( - handle_share_request_post, - handle_user_owned_request_listing, - handle_user_made_request_listing, handle_container_request_listing, - handle_user_share_request_delete, + handle_health_check, + handle_share_request_post, handle_user_add_token, handle_user_delete_token, handle_user_list_tokens, - handle_health_check, + handle_user_made_request_listing, + handle_user_owned_request_listing, + handle_user_share_request_delete, ) from swift_browser_ui.request.db import DBConn - logging.basicConfig(level=logging.DEBUG) asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) diff --git a/swift_browser_ui/sharing/api.py b/swift_browser_ui/sharing/api.py index 05d6e6997..be1401b18 100644 --- a/swift_browser_ui/sharing/api.py +++ b/swift_browser_ui/sharing/api.py @@ -3,8 +3,8 @@ import logging import os -import aiohttp.web +import aiohttp.web MODULE_LOGGER = logging.getLogger("api") MODULE_LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) @@ -24,7 +24,7 @@ async def has_access_handler(request: aiohttp.web.Request) -> aiohttp.web.Respon async def access_details_handler(request: aiohttp.web.Request) -> aiohttp.web.Response: """Handle access-details endpoint query.""" - access_details = dict() + access_details = {} access_details = await request.app["db_conn"].get_access_container_details( request.match_info["user"], request.query["owner"], @@ -56,7 +56,7 @@ async def gave_access_handler(request: aiohttp.web.Request) -> aiohttp.web.Respo async def shared_details_handler(request: aiohttp.web.Request) -> aiohttp.web.Response: """Handle shared-details endpoint query.""" - shared_details = dict() + shared_details = {} shared_details = await request.app["db_conn"].get_shared_container_details( request.match_info["owner"], request.match_info["container"] ) diff --git a/swift_browser_ui/sharing/bindings/__init__.py b/swift_browser_ui/sharing/bindings/__init__.py index e69de29bb..b3f58f027 100644 --- a/swift_browser_ui/sharing/bindings/__init__.py +++ b/swift_browser_ui/sharing/bindings/__init__.py @@ -0,0 +1 @@ +"""Async Python bindings for the swift-x-account-sharing backend.""" diff --git a/swift_browser_ui/sharing/bindings/bind.py b/swift_browser_ui/sharing/bindings/bind.py index 22d7a5c6b..c3327d396 100644 --- a/swift_browser_ui/sharing/bindings/bind.py +++ b/swift_browser_ui/sharing/bindings/bind.py @@ -2,15 +2,14 @@ import json -import typing import logging -import aiohttp - -import swift_browser_ui.common.signature - import ssl +import typing + +import aiohttp import certifi +import swift_browser_ui.common.signature ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) diff --git a/swift_browser_ui/sharing/db.py b/swift_browser_ui/sharing/db.py index d885f7c71..d5a9ca163 100644 --- a/swift_browser_ui/sharing/db.py +++ b/swift_browser_ui/sharing/db.py @@ -1,14 +1,13 @@ """Sharing backend database implementation.""" -import os import logging +import os import typing -import asyncio -import random import asyncpg +from swift_browser_ui.common.common_util import sleep_random MODULE_LOGGER = logging.getLogger("db") MODULE_LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) @@ -51,13 +50,13 @@ async def open(self) -> None: "Failed to establish connection. " "Pool will retry connection automatically.", ) - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() except asyncpg.exceptions.InvalidPasswordError: self.log.error("Invalid username or password for database.") - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() except asyncpg.exceptions.CannotConnectNowError: self.log.error("Database is not ready yet.") - await asyncio.sleep(random.randint(2, 5)) # nosec + await sleep_random() async def close(self) -> None: """Safely close the database connection.""" diff --git a/swift_browser_ui/sharing/server.py b/swift_browser_ui/sharing/server.py index bc4402b76..a05c7d1f9 100644 --- a/swift_browser_ui/sharing/server.py +++ b/swift_browser_ui/sharing/server.py @@ -1,35 +1,32 @@ """Sharing backend server module.""" -import sys -import logging import asyncio +import logging +import sys import typing import aiohttp.web import uvloop - -import swift_browser_ui.common.common_middleware import swift_browser_ui.common.common_handlers +import swift_browser_ui.common.common_middleware import swift_browser_ui.common.common_util - from swift_browser_ui.sharing.api import ( - has_access_handler, access_details_handler, - gave_access_handler, - shared_details_handler, - share_container_handler, delete_share_handler, edit_share_handler, + gave_access_handler, + handle_health_check, handle_user_add_token, handle_user_delete_token, handle_user_list_tokens, - handle_health_check, + has_access_handler, + share_container_handler, + shared_details_handler, ) from swift_browser_ui.sharing.db import DBConn - logging.basicConfig(level=logging.DEBUG) diff --git a/swift_browser_ui/ui/_convenience.py b/swift_browser_ui/ui/_convenience.py index 1fe5f0941..a8711685d 100644 --- a/swift_browser_ui/ui/_convenience.py +++ b/swift_browser_ui/ui/_convenience.py @@ -1,28 +1,24 @@ -""" -Miscallaneous convenience functions used during the project. +"""Miscallaneous convenience functions used during the project. Module contains funcions for e.g. authenticating against openstack v3 identity API, cache manipulation, cookies etc. """ -import secrets import logging +import secrets +import ssl import typing -import requests -import aiohttp_session import aiohttp import aiohttp.web +import aiohttp_session +import certifi +import requests import swift_browser_ui.common.signature - from swift_browser_ui.ui.settings import setd -import ssl -import certifi - - ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) @@ -69,8 +65,7 @@ def disable_cache( async def get_availability_from_token(token: str, client: aiohttp.ClientSession) -> dict: - """ - List available domains and projects for the unscoped token specified. + """List available domains and projects for the unscoped token specified. Params: token: str diff --git a/swift_browser_ui/ui/api.py b/swift_browser_ui/ui/api.py index 60da225ed..7305e6c9c 100644 --- a/swift_browser_ui/ui/api.py +++ b/swift_browser_ui/ui/api.py @@ -1,17 +1,15 @@ """Project functions for handling API requests from front-end.""" +import asyncio import re +import ssl import time import typing -import asyncio import urllib.parse import aiohttp.web import aiohttp_session - -import ssl import certifi - from swiftclient.utils import generate_temp_url from swift_browser_ui.ui._convenience import ( @@ -21,7 +19,6 @@ ) from swift_browser_ui.ui.settings import setd - ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) @@ -285,9 +282,9 @@ async def _swift_get_object_metadata_wrapper( meta = dict(filter(lambda i: "X-Object-Meta" in i[0], ret.headers.items())) meta = {k.replace("X-Object-Meta-", ""): v for k, v in meta.items()} if "s3cmd-attrs" in meta.keys(): - meta["s3cmd-attrs"] = { - k: v for k, v in [j.split(":") for j in meta["s3cmd-attrs"].split("/")] - } + meta["s3cmd-attrs"] = dict( + [j.split(":") for j in meta["s3cmd-attrs"].split("/")] + ) return (obj, meta) diff --git a/swift_browser_ui/ui/discover.py b/swift_browser_ui/ui/discover.py index 67026ea3e..b2c262e02 100644 --- a/swift_browser_ui/ui/discover.py +++ b/swift_browser_ui/ui/discover.py @@ -2,6 +2,7 @@ from typing import Union + import aiohttp.web from swift_browser_ui.ui.settings import setd diff --git a/swift_browser_ui/ui/front.py b/swift_browser_ui/ui/front.py index 6126d5b07..2c04d77f9 100644 --- a/swift_browser_ui/ui/front.py +++ b/swift_browser_ui/ui/front.py @@ -4,7 +4,6 @@ import aiohttp.web import aiohttp_session - from cryptography.fernet import InvalidToken from swift_browser_ui.ui.settings import setd @@ -64,7 +63,6 @@ async def index( request: typing.Optional[aiohttp.web.Request], ) -> typing.Union[aiohttp.web.Response, aiohttp.web.FileResponse]: """Serve the index page when running without a proxy.""" - try: if request is not None: session = await aiohttp_session.get_session(request) diff --git a/swift_browser_ui/ui/health.py b/swift_browser_ui/ui/health.py index 60086cd21..bde0babed 100644 --- a/swift_browser_ui/ui/health.py +++ b/swift_browser_ui/ui/health.py @@ -1,14 +1,13 @@ """Health check endpoint.""" -import typing import time +import typing import aiohttp.web from aiohttp.client_exceptions import ServerDisconnectedError import swift_browser_ui.common.signature - from swift_browser_ui.ui.settings import setd @@ -126,8 +125,8 @@ async def handle_health_check(request: aiohttp.web.Request) -> aiohttp.web.Respo "status": "Ok", } - services: typing.Dict[str, typing.Any] = dict() - performance: typing.Dict[str, typing.Any] = dict() + services: typing.Dict[str, typing.Any] = {} + performance: typing.Dict[str, typing.Any] = {} signature = swift_browser_ui.common.signature.sign_api_request("/health") api_params = { diff --git a/swift_browser_ui/ui/login.py b/swift_browser_ui/ui/login.py index 11f06a94c..1514323b8 100644 --- a/swift_browser_ui/ui/login.py +++ b/swift_browser_ui/ui/login.py @@ -1,10 +1,11 @@ """A module for handling the project login related tasks.""" -import time -import re import base64 import binascii +import re +import time +import typing # aiohttp import aiohttp.web @@ -12,15 +13,12 @@ from multidict import MultiDictProxy from oidcrp.exception import OidcServiceError -import typing - from swift_browser_ui.ui._convenience import ( disable_cache, get_availability_from_token, ) from swift_browser_ui.ui.settings import setd - HAKA_ENDPOINT = ( "{endpoint}/auth/OS-FEDERATION/identity_providers" "/haka/protocols/saml2/websso?origin={origin}" @@ -50,6 +48,7 @@ async def oidc_start(request: aiohttp.web.Request) -> aiohttp.web.Response: async def oidc_end(request: aiohttp.web.Request) -> aiohttp.web.Response: + """Finalize OIDC login and create a new session with the data from the OIDC provicer.""" # Response from AAI must have the query params `state` and `code` if "state" in request.query and "code" in request.query: request.app["Log"].debug("AAI response contained the correct params.") @@ -414,21 +413,17 @@ async def handle_project_lock(request: aiohttp.web.Request) -> aiohttp.web.Respo # Ditch all projects that aren't the one specified if project is defined if project in session["projects"]: - session["projects"] = { - k: v - for k, v in filter( + session["projects"] = dict( + filter( lambda val: val[0] == project, session["projects"].items(), ) - } + ) # If the project doesn't exist, allow all untainted projects else: - session["projects"] = { - k: v - for k, v in filter( - lambda val: not val[1]["tainted"], session["projects"].items() - ) - } + session["projects"] = dict( + filter(lambda val: not val[1]["tainted"], session["projects"].items()) + ) if not session["projects"]: session.invalidate() diff --git a/swift_browser_ui/ui/middlewares.py b/swift_browser_ui/ui/middlewares.py index bd983a51a..88bac8065 100644 --- a/swift_browser_ui/ui/middlewares.py +++ b/swift_browser_ui/ui/middlewares.py @@ -4,12 +4,11 @@ import time import typing -from aiohttp import web import aiohttp_session +from aiohttp import web from swift_browser_ui.ui.settings import setd - AiohttpHandler = typing.Callable[ [web.Request], typing.Coroutine[typing.Awaitable, typing.Any, web.Response] ] @@ -74,28 +73,18 @@ async def check_session_taintness( @web.middleware async def error_middleware(request: web.Request, handler: AiohttpHandler) -> web.Response: """Return the correct HTTP Error page.""" + to_process = {400, 401, 403, 404} + try: response = await handler(request) - if response.status == 400: - return return_error_response(400) - if response.status == 401: - return return_error_response(401) - if response.status == 403: - return return_error_response(403) - if response.status == 404: - return return_error_response(404) + if response.status in to_process: + return return_error_response(response.status) if response.status == 409: return response return response except web.HTTPException as ex: - if ex.status == 400: - return return_error_response(400) - if ex.status == 401: - return return_error_response(401) - if ex.status == 403: - return return_error_response(403) - if ex.status == 404: - return return_error_response(404) + if ex.status in to_process: + return return_error_response(ex.status) if ex.status == 409: raise ex if ex.status > 404 and ex.status < 500: diff --git a/swift_browser_ui/ui/server.py b/swift_browser_ui/ui/server.py index 391d57b17..eadb9801f 100644 --- a/swift_browser_ui/ui/server.py +++ b/swift_browser_ui/ui/server.py @@ -1,81 +1,76 @@ """swift_browser_ui server related convenience functions.""" # Generic imports +import asyncio +import base64 import logging import os -import sys -import asyncio +import secrets import ssl +import sys import typing -import secrets -import base64 - -import cryptography.fernet -import uvloop import aiohttp.web - import aiohttp_session import aiohttp_session.redis_storage - -from redis import asyncio as aioredis - +import cryptography.fernet +import uvloop from oidcrp.rp_handler import RPHandler +from redis import asyncio as aioredis +import swift_browser_ui.ui.middlewares +from swift_browser_ui.ui.api import ( + add_project_container_acl, + close_upload_session, + get_access_control_metadata, + get_crypted_upload_session, + get_os_user, + get_shared_container_address, + get_upload_session, + os_list_projects, + remove_container_acl, + remove_project_container_acl, + swift_create_container, + swift_delete_container, + swift_download_container, + swift_download_object, + swift_download_shared_object, + swift_get_metadata_container, + swift_get_project_metadata, + swift_list_containers, + swift_list_objects, + swift_replicate_container, + swift_update_container_metadata, +) +from swift_browser_ui.ui.discover import handle_discover from swift_browser_ui.ui.front import ( - index, browse, + index, loginpassword, select, - swjs, swasm, + swjs, ) +from swift_browser_ui.ui.health import handle_health_check from swift_browser_ui.ui.login import ( - oidc_start, - oidc_end, + credentials_login_end, handle_login, handle_logout, - sso_query_begin_oidc, + handle_project_lock, + oidc_end, + oidc_start, sso_query_begin, + sso_query_begin_oidc, sso_query_end, - credentials_login_end, - handle_project_lock, ) -from swift_browser_ui.ui.api import ( - get_crypted_upload_session, - swift_get_metadata_container, - swift_list_containers, - swift_list_objects, - swift_download_object, - swift_download_shared_object, - swift_download_container, - os_list_projects, - get_os_user, - get_access_control_metadata, - remove_container_acl, - add_project_container_acl, - get_shared_container_address, - swift_create_container, - swift_delete_container, - swift_replicate_container, - swift_update_container_metadata, - swift_get_project_metadata, - remove_project_container_acl, - get_upload_session, - close_upload_session, -) -from swift_browser_ui.ui.health import handle_health_check +from swift_browser_ui.ui.misc_handlers import handle_bounce_direct_access_request from swift_browser_ui.ui.settings import setd -import swift_browser_ui.ui.middlewares -from swift_browser_ui.ui.discover import handle_discover from swift_browser_ui.ui.signature import ( - handle_signature_request, handle_ext_token_create, handle_ext_token_list, handle_ext_token_remove, + handle_signature_request, ) -from swift_browser_ui.ui.misc_handlers import handle_bounce_direct_access_request - asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) @@ -91,7 +86,7 @@ async def kill_dload_client(app: aiohttp.web.Application) -> None: async def servinit( - inject_middleware: typing.List[typing.Any] = [], + inject_middleware: typing.Optional[typing.List] = None, ) -> aiohttp.web.Application: """Create an aiohttp server with the correct arguments and routes.""" middlewares = [ @@ -334,8 +329,7 @@ def run_server_secure( cert_file: str, cert_key: str, ) -> None: - """ - Run the server securely with a given ssl context. + """Run the server securely with a given ssl context. While this function is incomplete, the project is safe to run in production only via a TLS termination proxy with e.g. NGINX. diff --git a/swift_browser_ui/ui/settings.py b/swift_browser_ui/ui/settings.py index a7b6b8be8..56dfc2d79 100644 --- a/swift_browser_ui/ui/settings.py +++ b/swift_browser_ui/ui/settings.py @@ -1,5 +1,4 @@ -""" -Module containing all of the settings required in the global scope. +"""Module containing all of the settings required in the global scope. The different configurations are also listed here: @@ -37,8 +36,7 @@ import logging from os import environ -from typing import Union, Dict - +from typing import Dict, Union FORMAT = """\ [%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s] \ diff --git a/swift_browser_ui/ui/shell.py b/swift_browser_ui/ui/shell.py index 949dfa3ac..e3cc1140c 100644 --- a/swift_browser_ui/ui/shell.py +++ b/swift_browser_ui/ui/shell.py @@ -7,10 +7,9 @@ import click - from swift_browser_ui.__init__ import __version__ -from swift_browser_ui.ui.settings import setd, set_key, FORMAT -from swift_browser_ui.ui.server import servinit, run_server_insecure, run_server_secure +from swift_browser_ui.ui.server import run_server_insecure, run_server_secure, servinit +from swift_browser_ui.ui.settings import FORMAT, set_key, setd @click.group() diff --git a/swift_browser_ui/ui/signature.py b/swift_browser_ui/ui/signature.py index 2c926ed90..3f993e261 100644 --- a/swift_browser_ui/ui/signature.py +++ b/swift_browser_ui/ui/signature.py @@ -11,7 +11,6 @@ from .settings import setd - LOGGER = logging.getLogger("signature") diff --git a/swift_browser_ui/upload/api.py b/swift_browser_ui/upload/api.py index a7f68f2b9..2189c27b6 100644 --- a/swift_browser_ui/upload/api.py +++ b/swift_browser_ui/upload/api.py @@ -1,29 +1,29 @@ """API handlers for swift-upload-runner.""" -import logging -import typing -import aiohttp.web import asyncio import base64 import json +import logging import os +import typing + +import aiohttp.web from swift_browser_ui.common.vault_client import VaultClient from swift_browser_ui.upload.common import ( + VAULT_CLIENT, get_encrypted_upload_instance, get_session_id, get_upload_instance, parse_multipart_in, - VAULT_CLIENT, ) from swift_browser_ui.upload.download import ( - FileDownloadProxy, ContainerArchiveDownloadProxy, + FileDownloadProxy, ) from swift_browser_ui.upload.replicate import ObjectReplicationProxy - LOGGER = logging.getLogger(__name__) LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/upload/auth.py b/swift_browser_ui/upload/auth.py index a7b6b1a48..0a92a29f2 100644 --- a/swift_browser_ui/upload/auth.py +++ b/swift_browser_ui/upload/auth.py @@ -1,18 +1,17 @@ """Authorization handlers for swift-upload-runner.""" +import logging import os import secrets -import logging import aiohttp import aiohttp.web -import swift_browser_ui.common.types import swift_browser_ui.common.signature +import swift_browser_ui.common.types import swift_browser_ui.upload.common - LOGGER = logging.getLogger(__name__) @@ -85,8 +84,10 @@ async def handle_logout(request: aiohttp.web.Request) -> aiohttp.web.Response: # doesn't exist session_key: str = swift_browser_ui.upload.common.get_session_id(request) request.app.pop(session_key) - finally: - return aiohttp.web.Response(status=204) + except KeyError: + pass + + return aiohttp.web.Response(status=204) @aiohttp.web.middleware diff --git a/swift_browser_ui/upload/common.py b/swift_browser_ui/upload/common.py index 3e08dffc1..f8a240c33 100644 --- a/swift_browser_ui/upload/common.py +++ b/swift_browser_ui/upload/common.py @@ -1,16 +1,15 @@ """Common resources for swift-upload-runner.""" -import typing import logging import os +import typing import aiohttp.web import swift_browser_ui.upload.cryptupload as cryptupload from swift_browser_ui.upload import upload - LOGGER = logging.getLogger(__name__) LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/upload/cryptupload.py b/swift_browser_ui/upload/cryptupload.py index 5b9f9a01e..140018877 100644 --- a/swift_browser_ui/upload/cryptupload.py +++ b/swift_browser_ui/upload/cryptupload.py @@ -1,20 +1,18 @@ """Server upload propxy using aiohttp.""" +import asyncio import base64 -import os import logging -import asyncio -import typing +import os import secrets - import ssl -import certifi +import typing import aiohttp import aiohttp.web +import certifi import swift_browser_ui.upload.common as common - ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) @@ -82,6 +80,7 @@ def __init__( def check_header( self, ) -> bool: + """Return wether the header has been uploaded.""" return self.header_uploaded async def a_create_container(self) -> None: diff --git a/swift_browser_ui/upload/download.py b/swift_browser_ui/upload/download.py index 263c233fc..43ab06342 100644 --- a/swift_browser_ui/upload/download.py +++ b/swift_browser_ui/upload/download.py @@ -1,26 +1,24 @@ """Server object and container download handlers using aiohttp.""" +import asyncio +import logging import os import queue -import threading -import typing import tarfile +import threading import time -import asyncio -import logging +import typing import aiohttp.web - import requests from swift_browser_ui.upload.common import ( generate_download_url, - get_path_from_list, get_download_host, + get_path_from_list, ) - LOGGER = logging.getLogger(__name__) LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) diff --git a/swift_browser_ui/upload/replicate.py b/swift_browser_ui/upload/replicate.py index 24da25190..e7bb04b55 100644 --- a/swift_browser_ui/upload/replicate.py +++ b/swift_browser_ui/upload/replicate.py @@ -2,17 +2,16 @@ import logging -import typing import os -import aiohttp.web -import aiohttp.client - import ssl +import typing + +import aiohttp.client +import aiohttp.web import certifi from swift_browser_ui.upload import common - LOGGER = logging.getLogger(__name__) LOGGER.setLevel(os.environ.get("LOG_LEVEL", "INFO")) @@ -261,7 +260,6 @@ async def a_copy_from_container(self) -> None: self.source_host, container=self.source_container ) LOGGER.debug(f"Container url: {container_url}") - objects: typing.Union[typing.List, str] async with self.client.get( common.generate_download_url( self.source_host, @@ -275,7 +273,7 @@ async def a_copy_from_container(self) -> None: LOGGER.debug(f"Container fetch failed with status {resp.status}") raise aiohttp.web.HTTPBadRequest(reason="Source container fetch failed") LOGGER.debug("Got container object listing") - objects = await resp.text() - objects = objects.rstrip().lstrip().split("\n") - for i in objects: + objects_text = await resp.text() + objects_list = objects_text.rstrip().lstrip().split("\n") + for i in objects_list: await self.a_copy_object(i) diff --git a/swift_browser_ui/upload/server.py b/swift_browser_ui/upload/server.py index b3b4f2e14..a9da03ed0 100644 --- a/swift_browser_ui/upload/server.py +++ b/swift_browser_ui/upload/server.py @@ -1,39 +1,38 @@ """Server initialization functions.""" +import asyncio +import logging import os import sys -import logging -import asyncio import typing -import aiohttp.web import aiohttp.client - +import aiohttp.web import uvloop import swift_browser_ui.common.common_middleware import swift_browser_ui.common.common_util from swift_browser_ui.common.vault_client import VaultClient -from swift_browser_ui.upload.auth import ( - handle_login, - handle_logout, - handle_validate_authentication, -) from swift_browser_ui.upload.api import ( - handle_get_object, + handle_delete_project_whitelist, handle_get_container, + handle_get_object, + handle_health_check, + handle_object_header, handle_post_object_chunk, handle_post_object_options, - handle_health_check, - handle_upload_encrypted_object_options, - handle_whitelist_options, - handle_upload_encrypted_object, - handle_upload_encrypted_object_ws, handle_project_key, - handle_object_header, handle_project_whitelist, - handle_delete_project_whitelist, + handle_upload_encrypted_object, + handle_upload_encrypted_object_options, + handle_upload_encrypted_object_ws, + handle_whitelist_options, +) +from swift_browser_ui.upload.auth import ( + handle_login, + handle_logout, + handle_validate_authentication, ) from swift_browser_ui.upload.common import VAULT_CLIENT diff --git a/swift_browser_ui/upload/upload.py b/swift_browser_ui/upload/upload.py index f63ff9579..35744e0da 100644 --- a/swift_browser_ui/upload/upload.py +++ b/swift_browser_ui/upload/upload.py @@ -1,20 +1,18 @@ """Server object upload handlers using aiohttp.""" -import os -import typing import asyncio import logging +import os +import ssl +import typing -import aiohttp.web import aiohttp.client - -import ssl +import aiohttp.web import certifi from swift_browser_ui.upload import common - ssl_context = ssl.create_default_context() ssl_context.load_verify_locations(certifi.where()) diff --git a/swift_browser_ui_frontend/src/common/db.js b/swift_browser_ui_frontend/src/common/db.js index 76f64a08a..a1e936a3c 100644 --- a/swift_browser_ui_frontend/src/common/db.js +++ b/swift_browser_ui_frontend/src/common/db.js @@ -14,7 +14,7 @@ function initDB() { db.preferences.add({id: 1}); } }); - + return db; } diff --git a/swift_browser_ui_frontend/src/common/globalFunctions.js b/swift_browser_ui_frontend/src/common/globalFunctions.js index 79b5a6fd4..a8f369215 100644 --- a/swift_browser_ui_frontend/src/common/globalFunctions.js +++ b/swift_browser_ui_frontend/src/common/globalFunctions.js @@ -88,6 +88,3 @@ export function deleteTag (event, tag, currentTags) { event.preventDefault(); return currentTags.filter(el => el !== tag); } - - - diff --git a/swift_browser_ui_frontend/src/common/upload.js b/swift_browser_ui_frontend/src/common/upload.js index 95ded0a04..ddf37912e 100644 --- a/swift_browser_ui_frontend/src/common/upload.js +++ b/swift_browser_ui_frontend/src/common/upload.js @@ -346,4 +346,4 @@ export default class EncryptedUploadSession { this.handleMessage, {signal: this.signal}, ); } -} \ No newline at end of file +} diff --git a/swift_browser_ui_frontend/src/components/BrowserMainNavbar.vue b/swift_browser_ui_frontend/src/components/BrowserMainNavbar.vue index f431bba68..2ac60a1ba 100644 --- a/swift_browser_ui_frontend/src/components/BrowserMainNavbar.vue +++ b/swift_browser_ui_frontend/src/components/BrowserMainNavbar.vue @@ -47,7 +47,7 @@ + >
{{ item.title }} @@ -273,4 +273,4 @@ export default { } } - \ No newline at end of file + diff --git a/swift_browser_ui_frontend/src/components/CObjectTable.vue b/swift_browser_ui_frontend/src/components/CObjectTable.vue index ac1f23042..59312c3b1 100644 --- a/swift_browser_ui_frontend/src/components/CObjectTable.vue +++ b/swift_browser_ui_frontend/src/components/CObjectTable.vue @@ -255,7 +255,7 @@ export default { size: "small", title: "Edit tags", path: mdiPencilOutline, - onClick: ({ data }) => + onClick: ({ data }) => toggleEditTagsModal(data.name.value, null), onKeyUp: (event) => { if(event.keyCode === 13) { diff --git a/swift_browser_ui_frontend/src/components/CUploadButton.vue b/swift_browser_ui_frontend/src/components/CUploadButton.vue index ec76bf3ba..4defcf01d 100644 --- a/swift_browser_ui_frontend/src/components/CUploadButton.vue +++ b/swift_browser_ui_frontend/src/components/CUploadButton.vue @@ -2,7 +2,7 @@ Inspired by https://github.com/buefy/buefy/blob/3b3ae60e448ddfd669f20570d40812fd