diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml
index c88d6864..6f4acf46 100644
--- a/.github/workflows/on_release.yml
+++ b/.github/workflows/on_release.yml
@@ -61,6 +61,7 @@ jobs:
uses: docker/build-push-action@v2
with:
file: docker/Dockerfile
+ platforms: linux/amd64,linux/arm64
push: false
target: client
cache-from: type=registry,ref=permitio/opal-client:latest
@@ -76,6 +77,7 @@ jobs:
uses: docker/build-push-action@v2
with:
file: docker/Dockerfile
+ platforms: linux/amd64,linux/arm64
push: false
target: client-standalone
cache-from: type=registry,ref=permitio/opal-client-standalone:latest
@@ -91,6 +93,7 @@ jobs:
uses: docker/build-push-action@v2
with:
file: docker/Dockerfile
+ platforms: linux/amd64,linux/arm64
push: false
target: server
cache-from: type=registry,ref=permitio/opal-server:latest
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index 4117b475..87803b76 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -10,5 +10,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- - uses: actions/setup-python@v3
+ - uses: actions/setup-python@v4
+ with:
+ python-version: 3.x
- uses: pre-commit/action@v3.0.0
diff --git a/Makefile b/Makefile
index 30db38d1..6755b4d5 100644
--- a/Makefile
+++ b/Makefile
@@ -48,13 +48,13 @@ docker-build-client-standalone:
@docker build -t permitio/opal-client-standalone --target client-standalone -f docker/Dockerfile .
docker-run-client:
- @docker run -it -e "OPAL_SERVER_URL=$(OPAL_SERVER_URL)" -p 7000:7000 -p 8181:8181 permitio/opal-client
+ @docker run -it -e "OPAL_SERVER_URL=$(OPAL_SERVER_URL)" -p 7766:7000 -p 8181:8181 permitio/opal-client
docker-run-client-standalone:
@docker run -it \
-e "OPAL_SERVER_URL=$(OPAL_SERVER_URL)" \
-e "OPAL_POLICY_STORE_URL=$(OPAL_POLICY_STORE_URL)" \
- -p 7000:7000 \
+ -p 7766:7000 \
permitio/opal-client-standalone
docker-build-server:
diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml
index c865b280..466689a5 100644
--- a/docker/docker-compose-api-policy-source-example.yml
+++ b/docker/docker-compose-api-policy-source-example.yml
@@ -37,7 +37,7 @@ services:
# configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc).
# the data sources represents from where the opal clients should get a "complete picture" of the data they need.
# after the initial sources are fetched, the client will subscribe only to update notifications sent by the server.
- - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"]}]}}
+ - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}}
- OPAL_LOG_FORMAT_INCLUDE_PID=true
ports:
# exposes opal server on the host machine, you can access the server at: http://localhost:7002
@@ -53,8 +53,8 @@ services:
- OPAL_LOG_FORMAT_INCLUDE_PID=true
- OPAL_INLINE_OPA_LOG_FORMAT=http
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
- - "7000:7000"
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
+ - "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
# OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/
diff --git a/docker/docker-compose-example.yml b/docker/docker-compose-example.yml
index 8a690921..13855734 100644
--- a/docker/docker-compose-example.yml
+++ b/docker/docker-compose-example.yml
@@ -53,7 +53,7 @@ services:
# - opa_backup:/opal/backup:rw
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-git-webhook.yml b/docker/docker-compose-git-webhook.yml
index 8461318f..c5a39427 100644
--- a/docker/docker-compose-git-webhook.yml
+++ b/docker/docker-compose-git-webhook.yml
@@ -49,7 +49,7 @@ services:
- OPAL_LOG_FORMAT_INCLUDE_PID=true
- OPAL_INLINE_OPA_LOG_FORMAT=http
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-scopes-example.yml b/docker/docker-compose-scopes-example.yml
index 1722e925..789ebea5 100644
--- a/docker/docker-compose-scopes-example.yml
+++ b/docker/docker-compose-scopes-example.yml
@@ -34,7 +34,7 @@ services:
- OPAL_INLINE_OPA_LOG_FORMAT=http
- OPAL_SCOPE_ID=myscope
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-with-callbacks.yml b/docker/docker-compose-with-callbacks.yml
index c7bb2305..7ce3eacc 100644
--- a/docker/docker-compose-with-callbacks.yml
+++ b/docker/docker-compose-with-callbacks.yml
@@ -64,7 +64,7 @@ services:
- OPAL_OPA_HEALTH_CHECK_POLICY_ENABLED=True
# end of update callbacks config ---------------------------
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml
index 3a49c50f..156e665b 100644
--- a/docker/docker-compose-with-kafka-example.yml
+++ b/docker/docker-compose-with-kafka-example.yml
@@ -87,8 +87,8 @@ services:
- OPAL_LOG_FORMAT_INCLUDE_PID=true
- OPAL_INLINE_OPA_LOG_FORMAT=http
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
- - "7000:7000"
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
+ - "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
# OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/
diff --git a/docker/docker-compose-with-oauth-initial.yml b/docker/docker-compose-with-oauth-initial.yml
index 7524ac3e..98b647e8 100644
--- a/docker/docker-compose-with-oauth-initial.yml
+++ b/docker/docker-compose-with-oauth-initial.yml
@@ -60,7 +60,7 @@ services:
# Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file.
- ./docker_files/policy_test/authz.rego:/opal/authz.rego
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml
index b0d6d2e4..a0fd3bbc 100644
--- a/docker/docker-compose-with-rate-limiting.yml
+++ b/docker/docker-compose-with-rate-limiting.yml
@@ -51,7 +51,7 @@ services:
# Turns on rate limiting in the client (without this flag the client won't respect the server's rate limiting)
- OPAL_WAIT_ON_SERVER_LOAD=true
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7003
- "7003:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
@@ -72,7 +72,7 @@ services:
# Turns on rate limiting in the client (without this flag the client won't respect the server's rate limiting)
- OPAL_WAIT_ON_SERVER_LOAD=true
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7004
- "7004:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml
index 90db14e2..8a80ad2a 100644
--- a/docker/docker-compose-with-security.yml
+++ b/docker/docker-compose-with-security.yml
@@ -78,7 +78,7 @@ services:
- OPAL_AUTH_JWT_AUDIENCE=https://api.opal.ac/v1/
- OPAL_AUTH_JWT_ISSUER=https://opal.ac/
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml
index 32d22b06..0c15d5a9 100644
--- a/docker/docker-compose-with-statistics.yml
+++ b/docker/docker-compose-with-statistics.yml
@@ -51,7 +51,7 @@ services:
# turning on statistics reporting on the client side
- OPAL_STATISTICS_ENABLED=true
ports:
- # exposes opal client on the host machine, you can access the client at: http://localhost:7000
+ # exposes opal client on the host machine, you can access the client at: http://localhost:7766
- "7766:7000"
# exposes the OPA agent (being run by OPAL) on the host machine
# you can access the OPA api that you know and love at: http://localhost:8181
diff --git a/documentation/docs/FAQ.mdx b/documentation/docs/FAQ.mdx
index 23a0a9dd..14fe9d95 100644
--- a/documentation/docs/FAQ.mdx
+++ b/documentation/docs/FAQ.mdx
@@ -3,35 +3,69 @@ sidebar_position: 1
title: FAQ
---
-## Slack Community Questions
-
-:::note
-We often get **many questions in our Slack community**, which are relevant for a most OPAL users Please find a selection of them below.
-:::
+The following is a break down of common questions about OPAL by categories.
---
-### If I'm using OPAL just for Kubernetes level authorization, what benefits do I get from using OPAL compared to [kube-mgmt](https://github.com/open-policy-agent/kube-mgmt)?
-
-
- Kubernetes
-
-
-#### Answer:
+# Data updates
+
+## What is the difference between initial data load (baseline) and follow-on updates ?
+
+A baseline update is used to bring an OPAL-client from a no-state (first connect, reconnect) status, to a basic ready state.
+Follow on data update triggers continue to update the client state on top of a baseline.
+Data fetchers are used both to load the initial baseline data as indicated by [OPAL_DATA_CONFIG_SOURCES](/getting-started/running-opal/run-opal-server/data-sources#step-5-server-config---data-sources), and for subsequent update (or delta changes) as part of an [update event](/getting-started/quickstart/opal-playground/publishing-data-update)
+The instructions in `OPAL_DATA_CONFIG_SOURCES` should enable a client to reach the current needed relevant state at any point.
+
+## What is the difference between triggering an update event, and fetching the data for the event ?
+
+Triggering an update is done by calling the REST API on the opal-server, letting OPAL know there's an update. (You can also [publish events via the backbone pubsub (e.g. Kafka)](/tutorials/run_opal_with_kafka#triggering-events-directly-from-kafka) )
+The update will propagate between OPAL servers and then be sent to the relevant subscribing clients according to the topics it has.
+Once a client receives an update from a server, it will always use a data-fetcher to fetch the data it needs (unless the data is part of the update itself [which is less recommended]) according to the instructions in the event.
+The event via the config field also indicates which type of data-fetch-provider the client should use.
+
+## How does OPAL send the data itself for updates?
+
+Instead of sending the data itself, OPAL sends instructions to the subscribing clients (per-topic) on how to fetch the data directly.
+Note: since OPAL 0.3.0 you can also send data on the pub-sub event (but we don't recommend that as a default)
+Note: you can teach OPAL-clients to fetch data from different sources by writing a data-fetch-provider - small python modules that are easy to create
+
+## How are data update events handled for the interim period between an OPAL-client starting and it getting the baseline data?
+
+Once connected, the OPAL-client will start tracking data update events that you send and handle them. It does so
+from the start even before getting the first base data - and will process the events in order - so you don't need
+to track them specifically.
+
+## Does the OPAL-client fetch the update messages from the OPAL Server and re-run them after it builds the initial state?
+
+The client does not re-run, but more correctly spins tasks to update as events come, and processes them once possible.
+Usually loading the client is pretty fast, so we haven't encountered issues here. In general this is not a pure OPAL problem
+but a general multi process read/write sync challenge. If it is an issue for you, We can suggest a few options:
+
+- Create a clone source where clients get their base data set, instead of working directly with the rapidly changing data source (this can be another OPA+OPAL instance for example - almost like a master node)
+- Temporarily lock access to the database for writing until the client is ready - (reader/writer lock pattern)
+
+## How can I plug data into OPAL?
+
+OPAL is extensible, its plugin mechanism is called fetchers. With them you can pipe data into OPA from 3rd party sources.
+You can learn more about OPAL fetchers here or simply try to use an [existing one](https://github.com/permitio/opal-fetcher-postgres).
+
+## Which is the better mechanism to use for realtime? OPA http.fetch() or an OPAL Client fetch provider?
+
+Unless the data source is a very stable ingrained service (part of the core system), and the queries to it are simple
+(low data volume and frequency) - We would always recommend the data fetcher option. It decouples the problem of loading
+data from querying the policy.
+
+## How can we trigger the OPAL data fetch from within Rego?
+
+First of all, you are not really supposed to trigger it from Rego - the idea is usually for it to be async and decoupled
+from the policy itself. Instead, the data source or the service mutating it are the ones sending the triggers to OPAL,
+handling the updates in the background - independent of any policy queries.
+
+That being said, you can technically use `http.send()` to trigger an update - as the API trigger is restful.
+
+# Kubernetes
+
+## If I'm using OPAL just for Kubernetes level authorization, what benefits do I get from using OPAL compared to [kube-mgmt](https://github.com/open-policy-agent/kube-mgmt)?
OPAL solves the most pain for application-level authorization; where things move a lot faster than the pace of deployments or infrastructure configurations. But it also shines for usage with Kubernetes. We'll highlight three main points:
@@ -46,58 +80,26 @@ OPAL solves the most pain for application-level authorization; where things move
It may be possible to achieve a similar result with K8s Secrets, though there is no current way to do so with kube-mgmt. In addition this would tightly couple (potentially external) services into the k8s configuration.
-### Can I run the OPAL-server without `OPAL_BROADCAST_URI` ?
-
-
- Networking
-
-
-#### Answer:
-
-Yes, you can potentially choose to run it as a single instance with a single worker in which case you don’t need the broadcaster channel. The broadcaster, AKA the backbone pub-sub, is used to sync between opal-server instances, one instance doesn't require syncing of course.
-This is mainly useful for light-workloads (Where a single worker is enough) and development environments (where you just don't want the hassle of setting up a Kafka / Redis / Postgres service)
-
-### Does OPAL provide health-check routes ?
-
-
- Devops, maintenance and observability
-
-
-#### Answer:
+# Networking & Connectivity
-Yes both OPAL-server and OPAL-client serve a health check route at `"/healthcheck"`
+## Does OPAL handle reconnecting ?
+
+OPAL-clients constantly try to reconnect to their opal-servers, with an exponential back off.
+Once a client reconnects it will follow the data instructions in `OPAL_DATA_CONFIG_SOURCES` to again reach a baseline state for data.
+
+## How does OPAL handle data consistency / changes ?
+
+Short answer: with Speed.
+The pub/sub web socket is very lightweight and very fast - usually updates propagate within few milliseconds at worst.
+The updates themselves (by default) are also lightweight and contain only the event metadata (e.g. topic, data source), not the data itself.
+Hence OPAL-clients are aware of changes very quickly, and can react to the change (within their policy or healthchecks) even before they have the full data update.
+
+## Can I run the OPAL-server without `OPAL_BROADCAST_URI` ?
-### How does OPAL guarantee that the policy data is actually in sync with the actual application state. We get that OPAL server publishes updates to OPAL client but is there error correction in place.
+Yes. This is mainly useful for light-workloads (Where a single worker is enough) and development environments (where you just don't want the hassle of setting up a Kafka / Redis / Postgres service).
+Since server replication requires broadcasting, make sure that `UVICORN_NUM_WORKERS=1` and pod scaling is set to 1 (when running on k8s)
-#### Answer:
+## How does OPAL guarantee that the policy data is actually in sync with the actual application state. We get that OPAL server publishes updates to OPAL client but is there error correction in place.
Thanks to the lightweight nature of the OPAL pub/sub channel - clients very quickly learn of changes in the data state even before they have
the data itself - this allows the **OPAL-clients**, their **data-fetcher components**, their **managed OPA instances**
@@ -109,64 +111,27 @@ state. The simplest form of here is
You can read more about the mechanism **[here](/tutorials/healthcheck_policy_and_update_callbacks)**.
-### We are using the official OPAL postgres data fetcher. When the OPAL Client is being spun up, it will fetch the data from the postgres instance it is connected to. We are concerned that during that spin up phase, if any updates to the data source happen, how is it handled?
-
-
- Data Updates
-
-
-#### Answer:
+## How does OPAL handle the situation when an OPA container is restarted or gets out-of-sync with source system ?
-Once connected, the OPAL-client will start tracking data update events that you send and handle them. It does so
-from the start even before getting the first base data - and will process the events in order - so you don't need
-to track them specifically.
+[See discussion #385 on Github](https://github.com/permitio/opal/discussions/385)
-### Does the client fetch the update messages from the OPAL Server and re-run them after it builds the initial state?
+# Devops, maintenance and observability
-#### Answer:
+## Does OPAL provide health-check routes ?
-The client does not re-run, but more correctly spins tasks to update as events come, and processes them once possible.
-Usually loading the client is pretty fast, so we haven't encountered issues here. In general this is not a pure OPAL problem
-but a general multi process read/write sync challenge. If it is an issue for you, I can suggest a few options:
+Yes both OPAL-server and OPAL-client serve a health check route at `"/healthcheck"`
+Read more about [monitoring OPAL](/tutorials/monitoring_opal)
-- Create a clone source where clients get their base data set, instead of working directly with the rapidly changing data source (this can be another OPA+OPAL instance for example - almost like a master node)
-- Temporarily lock access to the database for writing until the client is ready - (reader/writer lock pattern)
+## Does OPAL provide some kind of monitoring dashboard ?
+
+Not yet - we are planing on releasing that as part of another major release of OPAL.
+If you want early access to those features , or want to contribute to their development - do let us know.|
+In the meanwhile, you can use something like [Prometheus](https://prometheus.io/) and feed it the /statistics (/tutorials/monitoring_opal)
+And of course you can use [Permit.io](https://permit.io) which adds many control plane interfaces on top of OPAL.
-### I'm configuring OPAL to run with a kafka backbone. We have an existing kafka setup using SASL authentication. I can't see any examples for configuring this with authentication. What's the best way to achieve this?
-
-
- Security
-
-
-#### Answer:
+# Security
+
+## I'm configuring OPAL to run with a kafka backbone. We have an existing kafka setup using SASL authentication. I can't see any examples for configuring this with authentication. What's the best way to achieve this?
At a glance it seems the current broadcaster Kafka backend doesn't support SASL. You can read more [here](https://github.com/permitio/broadcaster/blob/master/broadcaster/_backends/kafka.py).
Unless there is a way to pass this configuration via URL to the underlying Kafka consumer.
@@ -179,39 +144,19 @@ I would suggest one of the following:
- Use a Kafka proxy to add the needed params - e.g. [kafka-proxy](https://github.com/grepplabs/kafka-proxy).
- Open an issue on our GitHub requesting we do item #1 for you; and we will do our best to prioritize it.
-### How can I plug data into OPAL?
-
-#### Answer:
-
-OPAL is extensible, its plugin mechanism is called fetchers. With them you can pipe data into OPA from 3rd party sources.
-You can learn more about OPAL fetchers here or simply try to use an [existing one](https://github.com/permitio/opal-fetcher-postgres).
-
-### Which is the better mechanism to use for realtime? OPA http.fetch() or OPAL Client fetch module?
-
-#### Answer:
-
-Unless the data source is a very stable ingrained service (part of the core system), and the queries to it are simple
-(low data volume and frequency) - We would always recommend the data fetcher option. It decouples the problem of loading
-data from querying the policy.
-
-### How can we trigger the OPAL data fetch from within Rego?
-
-#### Answer:
-
-First of all, you are not really supposed to trigger it from Rego - the idea is usually for it to be async and decoupled
-from the policy itself. Instead, the data source or the service mutating it are the ones sending the triggers to OPAL,
-handling the updates in the background - independent of any policy queries.
-
-That being said, you can technically use `http.send()` to trigger an update - as the API trigger is restful.
+:::note
+Got a question that is missing from here? [**checkout our Slack community**](https://io.permit.io/community), it's a great place to ask more questions about OPAL.
+:::
-### How does OPAL handle the situation when an OPA container is restarted or gets out-of-sync with source system ?
+---
-#### Answer:
+# Case studies
-[See discussion #385 on Github](https://github.com/permitio/opal/discussions/385)
+## Handling a lot of data in OPA
-### I have generated a lot of permission data (e.g. 1.3mln records (or ~150MB)) and it causes the OPAL-client container memory to inflate to more than 2GB, and OPA crashes. Could you please advise what could be done to fix this?
+### Question
+I have generated a lot of permission data (e.g. 1.3mln records (or ~150MB)) and it causes the OPAL-client container memory to inflate to more than 2GB, and OPA crashes. Could you please advise what could be done to fix this?
I have started the containers with configuration similar to docker-compose-example, with corresponding data config source:
```
@@ -239,7 +184,7 @@ See the screenshot for the time difference between request processing.
I understand that this might not be a very optimal data structure, but still not really the performance I'd expect.
-#### Answer
+### Answer
These are most likely (>95%) issues with OPA not OPAL, as OPA is running with the OPAL-client container.
diff --git a/documentation/docs/getting-started/quickstart/docker-compose-config/opal-client.mdx b/documentation/docs/getting-started/quickstart/docker-compose-config/opal-client.mdx
index 1a05335c..c3e85d41 100644
--- a/documentation/docs/getting-started/quickstart/docker-compose-config/opal-client.mdx
+++ b/documentation/docs/getting-started/quickstart/docker-compose-config/opal-client.mdx
@@ -11,7 +11,7 @@ service:
- OPAL_LOG_FORMAT_INCLUDE_PID=true
- OPAL_INLINE_OPA_LOG_FORMAT=http
ports:
- - "7000:7000"
+ - "7766:7000"
- "8181:8181"
depends_on:
- opal_server
@@ -33,7 +33,7 @@ In our example `docker-compose.yml` file, OPA is enabled and runs on port `:8181
```yml showLineNumbers {3}
ports:
- - "7000:7000"
+ - "7766:7000"
- "8181:8181"
```
diff --git a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx
index 0f397872..1d11235e 100644
--- a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx
+++ b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx
@@ -39,7 +39,7 @@ services:
- OPAL_LOG_FORMAT_INCLUDE_PID=true
- OPAL_INLINE_OPA_LOG_FORMAT=http
ports:
- - "7000:7000"
+ - "7766:7000"
- "8181:8181"
depends_on:
- opal_server
diff --git a/documentation/docs/getting-started/quickstart/opal-playground/run-server-and-client.mdx b/documentation/docs/getting-started/quickstart/opal-playground/run-server-and-client.mdx
index 7517aa35..5b9b9c14 100644
--- a/documentation/docs/getting-started/quickstart/opal-playground/run-server-and-client.mdx
+++ b/documentation/docs/getting-started/quickstart/opal-playground/run-server-and-client.mdx
@@ -30,7 +30,7 @@ or you can continue with the hands-on tutorial.
**OPAL** (and also **OPA**) are now running on your machine. You should be aware of the following ports that are exposed on `localhost`:
- **OPAL Server** - PORT `:7002` - the **_OPAL client_** (and potentially the CLI) can connect to this port.
-- **OPAL Client** - PORT `:7000` - the **_OPAL client_** has its own API, but it's irrelevant to this tutorial.
+- **OPAL Client** - PORT `:7766` - the **_OPAL client_** has its own API, but it's irrelevant to this tutorial.
- **OPA** - PORT `:8181` - the port of the **_OPA Agent_** that is running **running in server mode**.
:::info
diff --git a/documentation/docs/getting-started/running-opal/run-opal-client/lets-run-the-client.mdx b/documentation/docs/getting-started/running-opal/run-opal-client/lets-run-the-client.mdx
index 6ef56b35..7f24249f 100644
--- a/documentation/docs/getting-started/running-opal/run-opal-client/lets-run-the-client.mdx
+++ b/documentation/docs/getting-started/running-opal/run-opal-client/lets-run-the-client.mdx
@@ -34,14 +34,14 @@ docker run -it \
--env OPAL_CLIENT_TOKEN \
--env OPAL_SERVER_URL \
--env OPAL_DATA_TOPICS \
- -p 7000:7000 \
+ -p 7766:7000 \
-p 8181:8181 \
permitio/opal-client
```
Please notice opal client exposes two ports when running opa inline:
-- OPAL Client (port `:7000`) - the OPAL client API (i.e: healthcheck, etc).
+- OPAL Client (port `:7766`) - the OPAL client API (i.e: healthcheck, etc).
- OPA (port `:8181`) - the port of the OPA agent (OPA is running in server mode).
#### 4) Run the container in production
diff --git a/documentation/docs/getting-started/running-opal/run-opal-server/data-sources.mdx b/documentation/docs/getting-started/running-opal/run-opal-server/data-sources.mdx
index e7afc069..a195686c 100644
--- a/documentation/docs/getting-started/running-opal/run-opal-server/data-sources.mdx
+++ b/documentation/docs/getting-started/running-opal/run-opal-server/data-sources.mdx
@@ -43,10 +43,10 @@ Let's break down this example value (check the [schema](https://github.com/permi
Each object in `entries` (schema: [DataSourceEntry](https://github.com/permitio/opal/blob/master/packages/opal-common/opal_common/schemas/data.py#L8)) is a **directive** that tells OPAL client to fetch the data and place it in OPA cache using the [Data API](https://www.openpolicyagent.org/docs/latest/rest-api/#data-api).
- **From where to fetch:** we tell OPAL client to fetch data from the [Permit.io API](https://api.permit.io/redoc) (specifically, from the `policy-config` endpoint).
-- **how to fetch (optional):** we can direct the client to use a specific configuration when fetching the data, for example here we tell the client to use a specific HTTP Authorization header with a bearer token in order to authenticate to the API.
+- **How to fetch (optional):** we can direct the client to use a specific configuration when fetching the data, for example here we tell the client to use a specific HTTP Authorization header with a bearer token in order to authenticate to the API.
- **Where to place the data in OPA cache:** although not specified, this entry uses the default of `/` which means at the root of OPA document hierarchy. You can specify another path with `dst_path` (check the [schema](https://github.com/permitio/opal/blob/master/packages/opal-common/opal_common/schemas/data.py#L8)).
- **Which clients should fetch:** the entry would be processed only by clients subscribed to one of the specified topics. (If unspecified, the default `policy_data` topic would be used).
-
+- **How often to fetch:** entry that has the `periodic_update_interval` value would be periodically pushed to clients with that interval (in secs), so they can keep refetching the most updated data from specified source.
#### Encoding this value in an environment variable:
You can use the python method of `json.dumps()` to get a one line string:
diff --git a/documentation/docs/overview/scopes.md b/documentation/docs/overview/scopes.md
index 01e2ba18..4ce2308e 100644
--- a/documentation/docs/overview/scopes.md
+++ b/documentation/docs/overview/scopes.md
@@ -103,7 +103,7 @@ docker run -it \
--env OPAL_SERVER_URL \
--env OPAL_DATA_TOPICS \
--env OPAL_SCOPE_ID=internal \
- -p 7000:7000 \
+ -p 7766:7000 \
-p 8181:8181 \
permitio/opal-client
```
diff --git a/documentation/docs/tutorials/track_an_api_bundle_server.mdx b/documentation/docs/tutorials/track_an_api_bundle_server.mdx
index c4bd061a..3452ab41 100644
--- a/documentation/docs/tutorials/track_an_api_bundle_server.mdx
+++ b/documentation/docs/tutorials/track_an_api_bundle_server.mdx
@@ -50,7 +50,7 @@ The `docker-compose.yml` we just downloaded ([Click here to view its contents](h
OPAL (and also OPA) are now running on your machine, the following ports are exposed on `localhost`:
- OPAL Server (port `:7002`) - the OPAL client (and potentially the cli) can connect to this port.
-- OPAL Client (port `:7000`) - the OPAL client has its own API, but it's irrelevant to this tutorial :)
+- OPAL Client (port `:7766`) - the OPAL client has its own API, but it's irrelevant to this tutorial :)
- OPA (port `:8181`) - the port of the OPA agent (running in server mode).
- OPA is being run by OPAL client in its container as a managed process.
- Nginx server that serves a static bundle file (bundle.tar.gz) on port 8000
diff --git a/documentation/package-lock.json b/documentation/package-lock.json
index 5573d0c7..45d490ba 100644
--- a/documentation/package-lock.json
+++ b/documentation/package-lock.json
@@ -7453,9 +7453,9 @@
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
},
"node_modules/json5": {
- "version": "2.2.1",
- "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",
- "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==",
+ "version": "2.2.3",
+ "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"bin": {
"json5": "lib/cli.js"
},
@@ -11901,9 +11901,9 @@
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/webpack": {
- "version": "5.74.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.74.0.tgz",
- "integrity": "sha512-A2InDwnhhGN4LYctJj6M1JEaGL7Luj6LOmyBHjcI8529cm5p6VXiTIW2sn6ffvEAKmveLzvu4jrihwXtPojlAA==",
+ "version": "5.76.1",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz",
+ "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==",
"dependencies": {
"@types/eslint-scope": "^3.7.3",
"@types/estree": "^0.0.51",
@@ -17866,9 +17866,9 @@
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
},
"json5": {
- "version": "2.2.1",
- "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz",
- "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA=="
+ "version": "2.2.3",
+ "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="
},
"jsonfile": {
"version": "6.1.0",
@@ -20994,9 +20994,9 @@
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"webpack": {
- "version": "5.74.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.74.0.tgz",
- "integrity": "sha512-A2InDwnhhGN4LYctJj6M1JEaGL7Luj6LOmyBHjcI8529cm5p6VXiTIW2sn6ffvEAKmveLzvu4jrihwXtPojlAA==",
+ "version": "5.76.1",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz",
+ "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==",
"requires": {
"@types/eslint-scope": "^3.7.3",
"@types/estree": "^0.0.51",
diff --git a/packages/opal-client/opal_client/cli.py b/packages/opal-client/opal_client/cli.py
index 1cfb0903..fd357e3f 100644
--- a/packages/opal-client/opal_client/cli.py
+++ b/packages/opal-client/opal_client/cli.py
@@ -14,7 +14,6 @@
from opal_common.cli.docs import MainTexts
from opal_common.cli.typer_app import get_typer_app
from opal_common.config import opal_common_config
-from opal_common.corn_utils import run_gunicorn, run_uvicorn
app = get_typer_app()
@@ -23,6 +22,7 @@
def run(engine_type: str = typer.Option("uvicron", help="uvicorn or gunicorn")):
"""Run the client as a daemon."""
typer.echo(f"-- Starting OPAL client (with {engine_type}) --")
+ from opal_common.corn_utils import run_gunicorn, run_uvicorn
if engine_type == "gunicorn":
app: FastAPI
@@ -63,7 +63,10 @@ def on_start(ctx: Context, **kwargs):
typer.echo(main_texts.docs)
opal_client_config.cli(
- [opal_common_config], typer_app=app, help=main_texts.docs, on_start=on_start
+ [opal_common_config],
+ typer_app=app,
+ help=main_texts.docs,
+ on_start=on_start,
)
diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py
index 1cf46458..c6404d0e 100644
--- a/packages/opal-client/opal_client/policy/updater.py
+++ b/packages/opal-client/opal_client/policy/updater.py
@@ -285,7 +285,7 @@ async def update_policy(
else bundle.deleted_files.dict()
)
logger.info(
- "got policy bundle (delta): '{diff_against_hash}' -> '{commit_hash}'",
+ "got policy bundle (delta): '{diff_against_hash}' -> '{commit_hash}', manifest: {manifest}, deleted: {deleted}",
commit_hash=bundle.hash,
diff_against_hash=bundle.old_hash,
manifest=bundle.manifest,
diff --git a/packages/opal-common/opal_common/fetcher/tests/http_fetch_test.py b/packages/opal-common/opal_common/fetcher/tests/http_fetch_test.py
index ec3ba172..1ffe4421 100644
--- a/packages/opal-common/opal_common/fetcher/tests/http_fetch_test.py
+++ b/packages/opal-common/opal_common/fetcher/tests/http_fetch_test.py
@@ -15,6 +15,7 @@
import pytest
import uvicorn
from fastapi import Depends, FastAPI, Header, HTTPException
+from flaky import flaky
from opal_common.fetcher import FetchingEngine
from opal_common.fetcher.providers.http_fetch_provider import HttpFetcherConfig
@@ -123,6 +124,7 @@ async def callback(data):
assert got_data_event.is_set()
+@flaky
@pytest.mark.asyncio
async def test_external_http_get():
"""Test simple http get on external (https://freegeoip.app/) site Checking
diff --git a/packages/requires.txt b/packages/requires.txt
index fb39a237..7d639a4b 100644
--- a/packages/requires.txt
+++ b/packages/requires.txt
@@ -9,3 +9,4 @@ pydantic[email]>=1.9.1,<2
typing-extensions;python_version<'3.8'
uvicorn[standard]>=0.17.6,<1
fastapi-utils>=0.2.1,<1
+setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/requirements.txt b/requirements.txt
index b3a6ff78..b9fb1d12 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ pytest-asyncio
flaky
wheel
twine
+setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability