Adds validation to fides keys, updates slugify to use underscores, closes #35

docs/fidesops/docs/guides/database_connectors.md

@@ -35,7 +35,7 @@ The connection between Fidesops and your database is represented by a _Connectio
 
 * `name`  is a  a human-readable name for your database.
 
-* `key`  is a string token that uniquely identifies your ConnectionConfig object. If you don't supply a `key`, the `name` value, converted to kebab-case (slugified), is used. For example, if the `name` is `Application PostgreSQL DB`, the converted key is `application-postgresql-db`.
+* `key`  is a string token that uniquely identifies your ConnectionConfig object. If you don't supply a `key`, the `name` value, converted to snake-case, is used. For example, if the `name` is `Application PostgreSQL DB`, the converted key is `application_postgresql_db`.
 
 * `connection-type` specifies the type of database. Valid values are `postgres`, `mongodb`, `mysql`, `redshift`, and `snowflake`.
 
@@ -52,7 +52,7 @@ PATCH api/v1/connection
 [
   { 
     "name": "Application PostgreSQL DB",
-    "key": "application-postgresql-db",
+    "key": "application_postgresql_db",
     "connection_type": "postgres",
     "access": "read"
   }
@@ -69,7 +69,7 @@ PATCH api/v1/connection
 [
   { 
     "name": "My Mongo DB",
-    "key": "my-mongo-db",
+    "key": "my_mongo_db",
     "connection_type": "mongodb",
     "access": "write"
   }
@@ -83,7 +83,7 @@ PATCH api/v1/connection
 [
   { 
     "name": "My MySQL DB",
-    "key": "my-mysql-db",
+    "key": "my_mysql_db",
     "connection_type": "mysql",
     "access": "write"
   }
@@ -124,7 +124,7 @@ This example sets the database secrets as a single `url` property, and skips the
 
 
 ```
-PUT api/v1/connection/my-mongo-db/secret?verify=false`
+PUT api/v1/connection/my_mongo_db/secret?verify=false`
 
 {
     "url": "mongodb://mongo_user:mongo_pass@mongodb_example/mongo_test"
@@ -148,7 +148,7 @@ you should adjust the ConnectionConfig Secrets properties through additional cal
 
 ```json
 {
-    "msg": "Test completed for ConnectionConfig with key: app-postgres-db.",
+    "msg": "Test completed for ConnectionConfig with key: app_postgres_db.",
     "test_status": "succeeded",
     "failure_reason": null
 }
@@ -158,7 +158,7 @@ you should adjust the ConnectionConfig Secrets properties through additional cal
 
 ```json
 {
-    "msg": "Secrets updated for ConnectionConfig with key: app-mongo-db.",
+    "msg": "Secrets updated for ConnectionConfig with key: app_mongo_db.",
     "test_status": "failed",
     "failure_reason": "Operation Failure connecting to MongoDB."
 }

docs/fidesops/docs/guides/policies.md

@@ -96,7 +96,7 @@ PATCH /api/v1/policy/{policy_key}/rule
 [
   {
     "name": "Mask Provided Emails",
-    "key": "mask-provided-emails",
+    "key": "mask_provided_emails",
     "action_type": "erasure",
     "masking_strategy": {
       "strategy": "hash",

docs/fidesops/docs/guides/storage.md

@@ -40,7 +40,7 @@ Let's get started. To create a new StorageConfig, use the following endpoint ([A
     "destinations": [
       {
         "name": str,
-        "key": str (optional),
+        "key": FidesOpsKey (optional),
         "type": str,
         "format": str
         "details": {
@@ -60,7 +60,7 @@ Let's get started. To create a new StorageConfig, use the following endpoint ([A
 Params:
 
 - `name`: A unique user-friendly name for your storage destination.
-- `key`: A unique key used to manage your storage destination. This is auto-generated from `name` (dasherized) if left blank.
+- `key`: A unique key used to manage your storage destination. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`.
 - `type`: Type of storage destination. Supported types include `s3`, `onetrust`, and `local`. You may configure multiple destinations of the same type.
 - `format`: Format of uploaded data. Supported formats include `json` and `csv`. For OneTrust and local destination types, use `json`.
 
@@ -95,7 +95,7 @@ Example response:
                 "naming": "request_id",
                 "object_name": "requests"
             },
-            "key": "s3-storage-2"
+            "key": "s3_storage_2"
         }
     ],
     "total": 1,

docs/fidesops/docs/tutorial/annotate_datasets.md

@@ -81,15 +81,15 @@ def create_dataset(connection_key, yaml_path, access_token):
 
 ### Call helper method to create a dataset
 
-Our connection_key is the `flaskr-postgres` ConnectionConfig we created in the previous step,
+Our connection_key is the `flaskr_postgres` ConnectionConfig we created in the previous step,
 and we're also passing in our completed YAML file:
 
 ```python
 if __name__ == "__main__":
     ...
     # Upload the Dataset YAML for our PostgreSQL schema
     datasets = create_dataset(
-        connection_key="flaskr-postgres",
+        connection_key="flaskr_postgres",
         yaml_path="fides_resources/flaskr_postgres_dataset.yml",
         access_token=access_token,
     )

docs/fidesops/docs/tutorial/define_policy.md

@@ -108,7 +108,7 @@ def delete_policy_rule(policy_key, key, access_token):
 For simplicity's sake, let's just create one Policy, one Rule, and one Target.
 
 Our single Policy will have one Rule with type `access`, meaning we just want to *retrieve* user data, not delete it. 
-We also configure on the Rule that any results will be uploaded to our local Storage `example-storage`.
+We also configure on the Rule that any results will be uploaded to our local Storage `example_storage`.
 
 Finally, we create a RuleTarget, that is looking for all data with the category `user.provided.identifiable` (and included subcategories). 
 
@@ -117,21 +117,21 @@ if __name__ == "__main__":
   ...
  # Create a Policy that returns all user data
     policy = create_policy(
-        key="example-request-policy",
+        key="example_request_policy",
         access_token=access_token,
     )
-    delete_policy_rule("example-request-policy", "access-user-data", access_token)
+    delete_policy_rule("example_request_policy", "access_user_data", access_token)
     create_policy_rule(
-        policy_key="example-request-policy",
-        key="access-user-data",
+        policy_key="example_request_policy",
+        key="access_user_data",
         action_type="access",
-        storage_destination_key="example-storage",
+        storage_destination_key="example_storage",
         access_token=access_token,
     )
 
     data_category = "user.provided.identifiable"
     create_policy_rule_target(
-        "example-request-policy", "access-user-data", data_category, access_token
+        "example_request_policy", "access_user_data", data_category, access_token
     )
 
 ```

docs/fidesops/docs/tutorial/execute_privacy_request.md

@@ -59,7 +59,7 @@ if __name__ == "__main__":
     email = "user@example.com"
     privacy_requests = create_privacy_request(
         email=email,
-        policy_key="example-request-policy",
+        policy_key="example_request_policy",
         access_token=access_token,
     )
 ```

docs/fidesops/docs/tutorial/postgres_connection.md

@@ -65,18 +65,18 @@ def configure_postgres_connection(
 
 ### Call helper methods to connect to Postgres
 
-Add calls for our new methods, to create a Postgres ConnectionConfig called `flaskr-postgres`, and 
+Add calls for our new methods, to create a Postgres ConnectionConfig called `flaskr_postgres`, and 
 then update that connection's secrets with individual URI components.  This will encrypt and save the URI components 
 and also attempt to make a test connection to our Flask App's Postgres Database.
 ```python
 
 if __name__ == "__main__":
     ...
     # Connect to our PostgreSQL database
-    create_postgres_connection(key="flaskr-postgres", access_token=access_token)
+    create_postgres_connection(key="flaskr_postgres", access_token=access_token)
 
     configure_postgres_connection(
-        key="flaskr-postgres",
+        key="flaskr_postgres",
         host=POSTGRES_SERVER,
         port=POSTGRES_PORT,
         dbname="flaskr",

docs/fidesops/docs/tutorial/storage_config.md

@@ -40,14 +40,14 @@ def create_local_storage(key, format, access_token):
 
 ### Call helper method to set up Storage
 
-This will define a local Storage location called `example-storage` that expects JSON data.  
+This will define a local Storage location called `example_storage` that expects JSON data.  
 
 ```python
     if __name__ == "__main__":
         ...
         # Configure a Storage Config to upload the results
         create_local_storage(
-            key="example-storage",
+            key="example_storage",
             format="json",
             access_token=access_token,
         )

src/fidesops/api/v1/endpoints/connection_endpoints.py

@@ -6,6 +6,7 @@
 from fastapi_pagination.ext.sqlalchemy import paginate
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
+from fidesops.schemas.shared_schemas import FidesOpsKey
 from pydantic import ValidationError, conlist
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_404_NOT_FOUND
@@ -70,7 +71,7 @@ def get_connections(
     response_model=ConnectionConfigurationResponse,
 )
 def get_connection_detail(
-    connection_key: str, db: Session = Depends(deps.get_db)
+    connection_key: FidesOpsKey, db: Session = Depends(deps.get_db)
 ) -> ConnectionConfig:
     """Returns connection configuration with matching key."""
     connection_config = ConnectionConfig.get_by(db, field="key", value=connection_key)
@@ -147,7 +148,7 @@ def patch_connections(
     status_code=204,
 )
 def delete_connection(
-    connection_key: str, *, db: Session = Depends(deps.get_db)
+    connection_key: FidesOpsKey, *, db: Session = Depends(deps.get_db)
 ) -> None:
     """Removes the connection configuration with matching key."""
     logger.info(f"Finding connection configuration with key {connection_key}")
@@ -218,7 +219,7 @@ def connection_status(
     response_model=TestStatusMessage,
 )
 async def put_connection_config_secrets(
-    connection_key: str,
+    connection_key: FidesOpsKey,
     *,
     db: Session = Depends(deps.get_db),
     unvalidated_secrets: connection_secrets_schemas,
@@ -260,7 +261,7 @@ async def put_connection_config_secrets(
     response_model=TestStatusMessage,
 )
 async def test_connection_config_secrets(
-    connection_key: str,
+    connection_key: FidesOpsKey,
     *,
     db: Session = Depends(deps.get_db),
 ) -> TestStatusMessage:

src/fidesops/api/v1/endpoints/dataset_endpoints.py

@@ -37,6 +37,7 @@
     ValidateDatasetResponse,
     BulkPutDataset,
 )
+from fidesops.schemas.shared_schemas import FidesOpsKey
 from fidesops.util.oauth_util import verify_oauth_client
 
 logger = logging.getLogger(__name__)
@@ -45,7 +46,7 @@
 
 # Helper method to inject the parent ConnectionConfig into these child routes
 def _get_connection_config(
-    connection_key: str, db: Session = Depends(deps.get_db)
+    connection_key: FidesOpsKey, db: Session = Depends(deps.get_db)
 ) -> ConnectionConfig:
     logger.info(f"Finding connection config with key '{connection_key}'")
     connection_config = ConnectionConfig.get_by(db, field="key", value=connection_key)
@@ -196,7 +197,7 @@ def get_datasets(
     response_model=FidesopsDataset,
 )
 def get_dataset(
-    fides_key: str,
+    fides_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
     connection_config: ConnectionConfig = Depends(_get_connection_config),
 ) -> FidesopsDataset:
@@ -226,7 +227,7 @@ def get_dataset(
     status_code=204,
 )
 def delete_dataset(
-    fides_key: str,
+    fides_key: FidesOpsKey,
     *,
     db: Session = Depends(deps.get_db),
     connection_config: ConnectionConfig = Depends(_get_connection_config),

src/fidesops/api/v1/endpoints/policy_endpoints.py

@@ -8,6 +8,7 @@
 )
 from fastapi_pagination.bases import AbstractPage
 from fastapi_pagination.ext.sqlalchemy import paginate
+from fidesops.schemas.shared_schemas import FidesOpsKey
 from pydantic import conlist
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
@@ -69,7 +70,7 @@ def get_policy_list(
 )
 def get_policy(
     *,
-    policy_key: str,
+    policy_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
 ) -> schemas.PolicyResponse:
     """
@@ -155,7 +156,7 @@ def create_or_update_rules(
         verify_oauth_client,
         scopes=[scopes.RULE_CREATE_OR_UPDATE],
     ),
-    policy_key: str,
+    policy_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
     input_data: conlist(schemas.RuleCreate, max_items=50) = Body(...),  # type: ignore
 ) -> schemas.BulkPutRuleResponse:
@@ -242,6 +243,16 @@ def create_or_update_rules(
             }
             failed.append(BulkUpdateFailed(**failure))
             continue
+        except ValueError as exc:
+            logger.warning(
+                f"Create/update failed for rule '{schema.key}' on policy {policy_key}: {exc}"
+            )
+            failure = {
+                "message": exc.args[0],
+                "data": dict(schema),
+            }
+            failed.append(BulkUpdateFailed(**failure))
+            continue
         else:
             created_or_updated.append(rule)
 
@@ -255,8 +266,8 @@ def create_or_update_rules(
 )
 def delete_rule(
     *,
-    policy_key: str,
-    rule_key: str,
+    policy_key: FidesOpsKey,
+    rule_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
 ) -> None:
     """
@@ -296,8 +307,8 @@ def create_or_update_rule_targets(
     client: ClientDetail = Security(
         verify_oauth_client, scopes=[scopes.RULE_CREATE_OR_UPDATE]
     ),
-    policy_key: str,
-    rule_key: str,
+    policy_key: FidesOpsKey,
+    rule_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
     input_data: conlist(schemas.RuleTarget, max_items=50) = Body(...),  # type: ignore
 ) -> schemas.BulkPutRuleTargetResponse:
@@ -389,9 +400,9 @@ def create_or_update_rule_targets(
 )
 def delete_rule_target(
     *,
-    policy_key: str,
-    rule_key: str,
-    rule_target_key: str,
+    policy_key: FidesOpsKey,
+    rule_key: FidesOpsKey,
+    rule_target_key: FidesOpsKey,
     db: Session = Depends(deps.get_db),
 ) -> None:
     """