Test signing (#50)

* first draft (for testing only) to sign job payload

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/rse-ops/accounting:app-latest
+FROM ghcr.io/rse-ops/pokemon:app-latest
 
 LABEL maintainer="Vanessasaurus <@vsoch>"
 
@@ -28,22 +28,9 @@ RUN python3 -m venv /env && \
 
 RUN mkdir -p /run/flux /var/lib/flux mkdir /etc/flux/system/cron.d /mnt/curve && \
     flux keygen /mnt/curve/curve.cert && \
-    # This probably needs to be done as flux user?
-    flux account create-db && \
-    flux account add-bank root 1 && \
-    flux account add-bank --parent-bank=root user_bank 1 && \
     # These need to be owned by flux
     chown -R flux /run/flux /var/lib/flux /mnt/curve && \
     # flux-imp needs setuid permission
     chmod u+s /usr/libexec/flux/flux-imp
-    # flux account add-user --username=fluxuser --bank=user_bank
-
-COPY ./example/multi-user/flux.service /etc/systemd/system/flux.service
-COPY ./example/multi-user/broker.toml /etc/flux/system/conf.d/broker.toml
-COPY ./example/multi-user/imp.toml /etc/flux/imp/conf.d/imp.toml
-
-RUN chmod 4755 /usr/libexec/flux/flux-imp \
- && chmod 0644 /etc/flux/imp/conf.d/imp.toml \
- && chmod 0644 /etc/flux/system/conf.d/broker.toml
 
 ENV PATH=/env/bin:${PATH}

.github/workflows/tests.yaml

@@ -48,4 +48,8 @@ jobs:
         export FLUX_REQUIRE_AUTH=true
         export TEST_AUTH=true
 
-        flux start pytest -xs tests/test_api.py
+        # This needs to be run as the flux instance owner
+        # E.g., if it's flux
+        # sudo -u flux flux start pytest -xs tests/test_api.py
+        whoami
+        sudo -u fluxuser flux start pytest -xs tests/test_api.py

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/flux-framework/flux-restful-api/tree/main) (0.0.x)
+ - Add better multi-user mode - running jobs on behalf of user (0.1.12)
  - Restore original rpc to get job info (has more information) (0.1.11)
  - Refactor of FLux Restful to use a database and OAauth2 (0.1.0)
  - Support for basic PAM authentication (0.0.11)

VERSION

@@ -1 +1 @@
-0.1.1
+0.1.12

app/core/security.py

@@ -13,21 +13,36 @@
 
 
 def create_access_token(
-    subject: Union[str, Any], expires_delta: timedelta = None
+    subject: Union[str, Any], expires_delta: timedelta = None, secret_key=None
 ) -> str:
+    """
+    Create a jwt access token.
+
+    We either use the user's secret key (which is hashed) or fall
+    back to the server set secret key.
+    """
+    # Use a user secret key, if they have one.
+    # Otherwise fall back to server secret key
+    secret_key = secret_key or settings.secret_key
     if expires_delta:
         expire = datetime.utcnow() + expires_delta
     else:
         expire = datetime.utcnow() + timedelta(
             minutes=settings.access_token_expires_minutes
         )
     to_encode = {"exp": expire, "sub": str(subject)}
-    return jwt.encode(to_encode, settings.secret_key, algorithm=ALGORITHM)
+    return jwt.encode(to_encode, secret_key, algorithm=ALGORITHM)
 
 
 def verify_password(plain_password: str, hashed_password: str) -> bool:
+    """
+    Verify the password
+    """
     return pwd_context.verify(plain_password, hashed_password)
 
 
 def get_password_hash(password: str) -> str:
+    """
+    Note we aren't providing a salt here, so the same password can generate different.
+    """
     return pwd_context.hash(password)

app/crud/base.py

@@ -29,6 +29,8 @@ def get(self, db: Session, id: Any) -> Optional[ModelType]:
     def get_multi(
         self, db: Session, *, skip: int = 0, limit: int = 100
     ) -> List[ModelType]:
+        if limit is None:
+            return db.query(self.model).offset(skip).all()
         return db.query(self.model).offset(skip).limit(limit).all()
 
     def create(self, db: Session, *, obj_in: CreateSchemaType) -> ModelType:

app/library/flux.py

@@ -2,23 +2,67 @@
 import os
 import re
 import shlex
+import subprocess
 import time
 
 import flux
 import flux.job
 
 from app.core.config import settings
 
+root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+submit_script = os.path.join(root, "scripts", "submit-job.py")
 
-def submit_job(handle, jobspec, user):
+
+class FakeJob:
+    def __init__(self, jobid):
+        self.jobid = jobid
+
+    def get_id(self):
+        return self.jobid
+
+
+def submit_job(handle, fluxjob, user):
     """
-    Handle to submit a job, either with flux job submit or on behalf of user.
+    Submit the job on behalf of user.
     """
     if user and hasattr(user, "user_name"):
         print(f"User submitting job {user.user_name}")
+        user = user.user_name
     elif user and isinstance(user, str):
         print(f"User submitting job {user}")
-    return flux.job.submit_async(handle, jobspec)
+
+    # If we don't have auth enabled, submit in single-user mode
+    if not settings.require_auth:
+        print("Submit in single-user mode.")
+        return flux.job.submit_async(handle, fluxjob)
+
+    # Update the payload for the correct user
+    # Use helper script to sign payload
+    payload = json.dumps(fluxjob.jobspec)
+    # payload['HOME'] =
+
+    # We ideally need to pipe the payload into flux python
+    try:
+        ps = subprocess.Popen(("echo", payload), stdout=subprocess.PIPE)
+        output = subprocess.check_output(
+            ("sudo", "-E", "-u", user, "flux", "python", submit_script),
+            stdin=ps.stdout,
+            env=os.environ,
+        )
+        ps.wait()
+
+    # A flux start without sudo -u flux can cause this
+    # This will be caught and returned to the user
+    except PermissionError as e:
+        raise ValueError(
+            f"Permission error: {e}! Are you running the instance as the flux user?"
+        )
+
+    jobid = output.decode("utf-8").strip()
+    print("Submit job {jobid}")
+    job = FakeJob(jobid)
+    return job
 
 
 def validate_submit_kwargs(kwargs, envars=None, runtime=None):
@@ -98,8 +142,9 @@ def prepare_job(user, kwargs, runtime=0, workdir=None, envars=None):
     # Set an attribute about the owning user
     if user and hasattr(user, "user_name"):
         fluxjob.setattr("user", user.user_name)
-    elif isinstance(user, str):
-        fluxjob.setattr("user", user)
+        user = user.user_name
+
+    fluxjob.setattr("user", user)
 
     # Set a provided working directory
     print(f"⭐️ Workdir provided: {workdir}")

app/models/user.py

@@ -8,6 +8,10 @@ class User(Base):
     id = Column(Integer, primary_key=True, index=True)
     user_name = Column(String, unique=True, index=True, nullable=False)
     hashed_password = Column(String, nullable=False)
+
+    # These are only used to encode and validate payloads, and
+    # can be reset at any time. Not related to the hashed password
+    secret_key = Column(String, nullable=True)
     is_active = Column(Boolean(), default=True)
     is_superuser = Column(Boolean(), default=False)
     jobs = relationship("Job", back_populates="owner")

app/routers/api.py

@@ -190,9 +190,10 @@ async def submit_job(request: Request, user=user_auth):
     include everything in this function instead of having separate
     functions.
     """
-    print(f"User for submit is {user}")
     from app.main import app
 
+    print(f"User for submit is {user}")
+
     # This can bork if no payload is provided
     try:
         payload = await request.json()

app/routers/views.py

@@ -163,7 +163,6 @@ async def submit_job_post(request: Request, user=user_auth):
     Receive data posted (submit) to the form.
     """
     print(user)
-    from app.main import app
 
     messages = []
     form = SubmitForm(request)
@@ -177,7 +176,7 @@ async def submit_job_post(request: Request, user=user_auth):
                 launcher.launch(form.kwargs, workdir=form.workdir, user=user)
             )
         else:
-            return submit_job_helper(request, app, form, user=user)
+            return submit_job_helper(request, form, user=user)
     else:
         print("🍒 Submit form is NOT valid!")
     return templates.TemplateResponse(
@@ -192,10 +191,12 @@ async def submit_job_post(request: Request, user=user_auth):
     )
 
 
-def submit_job_helper(request, app, form, user):
+def submit_job_helper(request, form, user):
     """
     A helper to submit a flux job (not a launcher)
     """
+    from app.main import app
+
     # Submit the job and return the ID, but allow for error
     # Prepare the flux job! We don't support envars here yet
     try:

app/scripts/sign-job.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# See https://github.com/flux-framework/flux-core/blob/master/t/t2404-job-exec-multiuser.t#L48
+# for an example of using this. This should be run with flux python, as done in library/flux.py
+import sys
+
+from flux.security import SecurityContext
+
+if len(sys.argv) < 2:
+    print("Usage: {0} USERID".format(sys.argv[0]))
+    sys.exit(1)
+
+userid = int(sys.argv[1])
+ctx = SecurityContext()
+payload = sys.stdin.read()
+
+print(ctx.sign_wrap_as(userid, payload, mech_type="munge").decode("utf-8"))

app/scripts/submit-job.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+import sys
+
+import flux
+import flux.job
+
+payload = sys.stdin.read()
+fluxjob = flux.job.JobspecV1.from_yaml_stream(payload)
+job = flux.job.submit_async(flux.Flux(), payload)
+print(job.get_id())

clients/python/flux_restful_client/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.1"
+__version__ = "0.1.12"
 AUTHOR = "Vanessa Sochat"
 EMAIL = "vsoch@users.noreply.github.com"
 NAME = "flux-restful-client"

docs/getting_started/developer-guide.md

@@ -363,7 +363,14 @@ And open your browser to `localhost:9999`
 ### Run Tests
 
 To run tests, from within the devcontainers environment (or with a local install)
-of Flux alongside the app) you can do:
+of Flux alongside the app) you can use flux start. You will need to run them as the
+flux instance owner. E.g., if it's flux:
+
+```bash
+$ sudo -u flux flux start pytest -xs tests/test_api.py
+```
+
+or if it's just root / a single user:
 
 ```bash
 $ flux start pytest -xs tests/test_api.py

docs/getting_started/user-guide.md

@@ -9,36 +9,75 @@ The Python client is also included in our [Tutorials](https://flux-framework.org
 
 ## How does it work?
 
+### Modes
+
+There are two modes of interaction:
+
+ - **single-user mode**: assumes you are running an instance as the flux user, and submitting jobs as that user.
+ - **multi-user mode**: requires authentication via the RESTful API with an encoded payload to request expiring tokens. When authentication is successful, the
+   job is run as the same user on the system on behalf of the flux user.
+
+### Authentication
+
 If you choose to deploy without authentication, this is a ⚠️ proceed at your own risk ⚠️ sort of deal.
-By default it's not required, but it's highly recommended. To require authentication, we set a few
-environment variables to turn it on and define credentials and a secret (e.g., a driver that
-is running the API might randomly generate these accounts and secret) and then all interactions
-with the API or interface require authenticating. In the case of the web interface, we fall back
-to basic auth, and the user needs to enter a username and password. In the case of the API,
-we taken an OAuth2 based approach, where a request will original return with a 401 status
-and the "www-authenticate" header, and the calling client needs to then prepare an encoded
+We call this the "single-user" case, and it means that you are submitting jobs as the instance owner,
+typically a user named "flux." If it's just you that owns the cluster, or a small group of trusted friends,
+this is probably OK. When you enable authentication, the following happens:
+
+ - A server secret that you export via `FLUX_SECREY_KEY` is used to encode payloads. You'll need to provide this to users.
+ - The server is created adding users with names and passwords, so every user known to Flux Restful is known to the server.
+   - Passwords are hashed
+   - We don't currently check authentication here with PAM (but we could).
+ - A user making a request provided an encoded payload (first) with the encoded username and password
+ - The server decodes the payload, authenticates, and (given a valid username and password) generates an expiring token.
+ - The user adds the token header to subsequent requests.
+
+To require this authentication, we set a few environment variables to turn it on and define credentials
+and a secret (e.g., a driver that is running the API might randomly generate these accounts and secret) and then all interactions
+with the API or interface require authenticating. As an example, the Flux Operator will make both the server user
+accounts and the Flux Restful database accounts when you spin up a MiniCluster.
+
+#### Web Interface Basic Authentication
+
+In the case of the web interface (which does not necessarily need to be exposed, e.g., the Flux Operator requires a port forward)
+we fall back to basic auth, and the user needs to enter a username and password.
+
+#### API OAuth2 Style Authentication
+
+In the case of the API, we taken an OAuth2 based approach, where a request will originally
+return with a 401 status and the "www-authenticate" header, and the calling client needs to then prepare an encoded
 payload to request a token. A successful receipt of the payload will return the token,
-which can be added to an Authorization header for subsequent requests.
+which can be added to an Authorization header for subsequent requests (up until it expires).
 
 You largely don't need to worry about the complexity of the above because the SDKs will
 handle these interactions for you, given that you've provided some credentials and secret key.
 If you are using the Flux Operator, you largely don't need to do anything, as it will
 generate and provide both.
 
+## What does this user-guide include?
+
+This user-guide assumes you are a user of the flux restful API, meaning you are either
+running it alongside a production Flux cluster, or it's running already in another
+context and you have been given credentials to access it. If you want to learn about how
+to setup the API itself, see the [developer documentation](developer-guide.md).
+
+
 ## Environment
 
-You should either have a Flux user, token, and secret key from the server you created, or provided to you
-by an administrator. E.g.,
+Whether you are in single- or multi- user mode, you will need a username and token to
+interact with the server. In single-user mode this will be the flux superuser credentials
+that the server was created with. In multi-user mode this will be your username and password,
+along with a secret key to encode paylods for the token. E.g.,
 
 ```bash
 $ export FLUX_USER=fluxuser
 $ export FLUX_TOKEN=12345
 $ export FLUX_SECRET_KEY=notsecrethoo
 ```
 
-You really should only be interacting with a server that doesn't require authentication if you are a developer.
 From here, continue reading the user guide for different language clients,
-or see our Python [examples](https://github.com/flux-framework/flux-restful-api/tree/main/clients/python/examples) folder.
+or see our Python [examples](https://github.com/flux-framework/flux-restful-api/tree/main/clients/python/examples) folder
+for snippet examples, or the [tutorials](../tutorials/index.md) for more complex setups.
 
 ## Python