Merge pull request #19 from reallistic/bugfix/18/allow_disable_gzip

Bugfix/18/allow disable gzip
reallistic · Apr 26, 2021 · e646f30 · e646f30
2 parents 197cc76 + fab97fd
commit e646f30
Show file tree

Hide file tree

Showing 9 changed files with 192 additions and 76 deletions.
diff --git a/README.md b/README.md
@@ -104,6 +104,8 @@ optional arguments:
 If pyloot detects it is running in a multiprocessing environment with an inmemory backend
 it will refuse to serve the webpages/requests.
 
+This is common for gunicorn servers running with multiple workers. If you run pyloot embedded in a gunicorn server with multiple workers statistics will be collected in each individual worker and a random worker will be selected when returning statistics. This can still give an idea of what is going on but is considered in accurate.
+
 The WSGIMiddleware of starlette sets `environ["wsgi.multiprocess"]=True` regardless of the server.
 This can by bypassed with a wrapper **use with caution**:
 
@@ -119,6 +121,36 @@ def pyloot_wrapper(wsgi_environ, start_response):
 app.mount("/_pyloot", WSGIMiddleware(pyloot_wrapper))
 ```
 
+# Disabling gzip encoding
+By default, the pyloot server will gzip encode the response metadata.
+If pyloot is running behind a middleware that gzip encodes data the encoding could happen twice.
+This will result in the following error being shown in the UI:
+
+```text
+Error parsing the response data. Check the server logs. If everything looks ok, you make need to disable gzip in pyloot. For more info see the README.
+```
+
+To disable gzip encoding do the following:
+
+```python
+from pyloot import PyLoot
+from pyloot import PyLootServer
+
+pyloot = PyLoot(server=PyLootServer(disable_response_gzip=True))
+```
+
+
+If a remote server is used, it must be configured directly on the server like so:
+
+```python
+from pyloot import PyLoot
+from pyloot import PyLootServer
+from pyloot import HTTPRemoteBackend
+
+backend = HTTPRemoteBackend(host="127.0.0.1", port=8000)
+pyloot = PyLoot(server=PyLootServer(backend=backend, disable_response_gzip=True))
+```
+
 # Screenshots
 ### View history of object counts by object group:
 ![history screenshot](https://raw.githubusercontent.com/reallistic/pyloot/master/docs/history.png)

diff --git a/package-lock.json b/package-lock.json
diff --git a/pyloot-web/ListPage.jsx b/pyloot-web/ListPage.jsx
@@ -22,7 +22,7 @@ export function ListPage({
   if (isNaN(pageLimit)) {
     pageLimit = DEFAULT_PAGE_LIMIT;
   }
-  const [apiData, loading] = useDataLoader(url, [], true);
+  const [apiData, apiError, loading] = useDataLoader(url, [], true);
   const [page, setPage] = useState(0);
   const onPageUpdate = useMemo(() => {
     return (nextPage) => {
@@ -70,8 +70,9 @@ export function ListPage({
   const end = Math.min(start + pageLimit, data.length);
   const dataSlice = data.slice(start, end);
 
+  let showError = apiError !== null && !loading;
   let showLoading = loading && data.length === 0;
-  let showEmpty = !loading && dataSlice.length === 0;
+  let showEmpty = apiError === null && !loading && dataSlice.length === 0;
   return (
     <section className="list-page">
         <progress
@@ -120,6 +121,16 @@ export function ListPage({
       {showEmpty ? (
         <span className="loading">{emptyMessage || "Empty history"}</span>
       ) : null}
+      {showError ? (
+        <article className="message is-danger">
+          <div className="message-header">
+            <p>Error</p>
+          </div>
+          <div className="message-body">
+            {apiError}
+          </div>
+        </article>
+      ) : null}
       <div className={listContainerClassName}>
         {dataSlice.map((item, index) =>
           React.createElement(listItemComponent, {

diff --git a/pyloot-web/ObjectPage.jsx b/pyloot-web/ObjectPage.jsx
@@ -73,7 +73,7 @@ export function ObjectPage() {
             </div>
           </div>
         </div>
-        <code>{objectItemFetchError.text}</code>
+        <code>{objectItemFetchError}</code>
       </section>
     );
   }

diff --git a/pyloot-web/dataLoader.js b/pyloot-web/dataLoader.js
@@ -1,6 +1,6 @@
 import { useEffect, useState } from "react";
 
-import { fetchJson } from "./utils";
+import { fetchJson, getErrorMessage } from "./utils";
 
 export function useDataLoader(url, initialData, initialLoading) {
   if (initialData === undefined) {
@@ -12,16 +12,21 @@ export function useDataLoader(url, initialData, initialLoading) {
 
   const [loading, setLoading] = useState(initialLoading);
   const [data, setData] = useState(initialData);
+  const [error, setError] = useState(null);
   useEffect(() => {
     setLoading(initialLoading);
     setData(initialData);
     const dataLoader = new DataLoader(
       url,
       (data) => {
+        setError(null);
         setData(data);
         setLoading(false);
       },
-      () => setLoading(false),
+      (error) => {
+          setLoading(false);
+          setError(error);
+      },
       () => setLoading(true)
     );
     dataLoader.start();
@@ -31,7 +36,7 @@ export function useDataLoader(url, initialData, initialLoading) {
     };
   }, [url]);
 
-  return [data, loading];
+  return [data, error, loading];
 }
 
 export class DataLoader {
@@ -50,6 +55,10 @@ export class DataLoader {
     console.log(`<DataLoader url=${this.url}>`, ...messages);
   }
 
+  error(...messages) {
+    console.error(`<DataLoader url=${this.url}>`, ...messages);
+  }
+
   fetchData() {
     if (this.loading) {
       this.log("Already loading data");
@@ -59,11 +68,17 @@ export class DataLoader {
     this.log("Loading data");
     this.loading = true;
     this.onLoading && this.onLoading();
-    return fetchJson(this.url).then((data) => {
-      this.loading = false;
-      this.data = data;
-      return data;
-    });
+    return fetchJson(this.url).then(
+      (data) => {
+        this.loading = false;
+        this.data = data;
+        return data;
+      },
+      (error) => {
+        this.loading = false;
+        throw error;
+      }
+    );
   }
 
   start() {
@@ -94,8 +109,9 @@ export class DataLoader {
       },
       (error) => {
         if (this.active) {
-          this.log("retrying data fetch later");
-          this.onError && this.onError(error);
+          const errorMessage = getErrorMessage(error);
+          this.error("retrying data fetch later", errorMessage);
+          this.onError && this.onError(errorMessage);
           this.timer = setTimeout(this.update.bind(this), 30 * 1000);
         }
       }

diff --git a/pyloot-web/utils.js b/pyloot-web/utils.js
@@ -7,16 +7,29 @@ export function sorted(arr, compare) {
 
 export function fetchJson(url) {
   return fetch(url)
-    .then((response) => {
-      return response.text().then((responseText) => {
-        try {
-          const data = JSON.parse(responseText);
-          return { response, data, text: responseText };
-        } catch (e) {
-          return { response, text: responseText, data: null };
-        }
-      });
-    })
+    .then(
+      (response) => {
+        return response.text().then((responseText) => {
+          try {
+            const data = JSON.parse(responseText);
+            return { response, data, text: responseText };
+          } catch (e) {
+            if (response.ok) {
+              const text = (
+                "Error parsing the response data. Check the server logs. " +
+                "If everything looks ok, you make need to disable gzip in pyloot. " +
+                "For more info see the README."
+              );
+              throw { response, text, data: null };
+            }
+            return { response, text: responseText, data: null };
+          }
+        });
+      },
+      (error) => {
+        throw { statusText: error.message, text: error.message, data: null };
+      }
+    )
     .then(({ response, data, text }) => {
       if (!response.ok) {
         throw { statusText: response.statusText, data, text };
@@ -108,9 +121,26 @@ export function useApi(url, initialData) {
       },
       (error) => {
         setLoading(false);
-        setError(error);
+        setError(getErrorMessage(error));
       }
     );
   }, [url]);
   return [data, error, loading];
 }
+
+
+export function getErrorMessage(obj) {
+  let errorMessage = "Unknown error";
+  if (obj instanceof Error) {
+    errorMessage = obj.toString();
+  }
+  else if (obj instanceof Object) {
+    if (Object.hasOwnProperty.call(obj, "text")) {
+      errorMessage = obj.text;
+    }
+  }
+  else if (typeof obj === "string" ) {
+    errorMessage = obj;
+  }
+  return errorMessage;
+}
diff --git a/pyloot/__init__.py b/pyloot/__init__.py
@@ -22,7 +22,12 @@ def __init__(
         port: Optional[int] = None,
         backend: Optional[BaseBackend] = None,
         interval: int = 30,
+        server: Optional[PyLootServer] = None,
     ):
+        if server:
+            if backend:
+                logger.warning("ignoring backend since server is present")
+            backend = server.get_backend()
         if backend and (host or port):
             logger.warning("ignoring host and port since backend is present")
         if backend:
@@ -34,7 +39,7 @@ def __init__(
 
         self._running = False
         self._thread_ended = threading.Event()
-        self._server: Optional[PyLootServer] = None
+        self._server: Optional[PyLootServer] = server
         self._interval = interval
 
     def start(self):