Improve local mode (#361)

README.md

@@ -11,22 +11,23 @@
 
 # HyperDX
 
-[HyperDX](https://hyperdx.io) helps engineers figure out why production is
-broken faster by centralizing and correlating logs, metrics, traces, exceptions
+[HyperDX](https://hyperdx.io) helps engineers quickly figure out why production is
+broken by centralizing and correlating logs, metrics, traces, exceptions
 and session replays in one place. An open source and developer-friendly
 alternative to Datadog and New Relic.
 
 <p align="center">
-  <a href="https://www.hyperdx.io/docs">Documentation</a> • <a href="https://discord.gg/FErRRKU78j">Chat on Discord</a>  • <a href="https://api.hyperdx.io/login/demo">Live Demo</a>  • <a href="https://github.com/hyperdxio/hyperdx/issues/new">Bug Reports</a> • <a href="./CONTRIBUTING.md">Contributing</a>
+  <a href="https://www.hyperdx.io/docs">Documentation</a> • <a href="https://hyperdx.io/discord">Chat on Discord</a>  • <a href="https://api.hyperdx.io/login/demo">Live Demo</a>  • <a href="https://github.com/hyperdxio/hyperdx/issues/new">Bug Reports</a> • <a href="./CONTRIBUTING.md">Contributing</a>
 </p>
 
 - 🕵️ Correlate end to end, go from browser session replay to logs and traces in
   just a few clicks
 - 🔥 Blazing fast performance powered by Clickhouse
 - 🔍 Intuitive full-text search and property search syntax (ex. `level:err`)
+- ⏱️ Monitor health and performance from HTTP requests to DB queries (APM)
 - 🤖 Automatically cluster event patterns from billions of events
 - 📈 Dashboard high cardinality events without a complex query language
-- 🔔 Set up alerts in just a few clicks
+- 🔔 Set up alerts on logs, metrics, or traces in just a few clicks
 - `{` Automatic JSON/structured log parsing
 - 🔭 OpenTelemetry native
 

docker-compose.dev.yml

@@ -241,7 +241,7 @@ services:
     ports:
       - 8080:8080
     environment:
-      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
+      NEXT_PUBLIC_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
       NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'

docker-compose.yml

@@ -189,6 +189,7 @@ services:
     ports:
       - ${HYPERDX_APP_PORT}:${HYPERDX_APP_PORT}
     environment:
+      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000'
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       NODE_ENV: development
       PORT: ${HYPERDX_APP_PORT}

docker/local/Dockerfile

@@ -49,6 +49,7 @@ COPY --from=app ./styles ./styles
 
 ENV NEXT_TELEMETRY_DISABLED 1
 ENV NEXT_OUTPUT_STANDALONE true
+ENV NEXT_PUBLIC_IS_LOCAL_MODE true
 RUN yarn build
 
 # == Clickhouse/Base Image ==

docker/local/README.md

@@ -3,8 +3,8 @@
 HyperDX Local is a single container local-optimized version of HyperDX that allows you to pipe OpenTelemetry telemetry (logs, metrics, traces) to a local instance of HyperDX running on your own machine. This makes it easily to debug complex applications locally using the same telemetry you have in prod or to test your instrumentation before pushing it into production.
 
 HyperDX Local has a few additional benefits over the regular open source version:
-- 📦 Packaged in a single container, to slot alongside your existing development environment
-- 🔑 No need to create an account or login
+- 📦 Packaged in a single container, to slot alongside your existing dev environment (ex. Docker Compose stack)
+- 🔑 No need to auth to send and view telemetry
 - 🐏 Optimized for lower memory footprint
 
 And it has all the features you would expect from HyperDX:
@@ -24,7 +24,7 @@ docker run -p 8000:8000 -p 4318:4318 -p 4317:4317 -p 8080:8080 -p 8002:8002 TODO
 
 Afterwards, you can visit `http://localhost:8080` and immediately jump into the HyperDX UI.
 
-> We recommend having at least 1GB of RAM and 1 CPU core available for the container.
+> We recommend having _at least_ 1GB of RAM and 1 CPU core available for the container.
 
 ## Configuring Instrumentation
 
@@ -35,18 +35,37 @@ Configuring instrumentation for HyperDX local is similar to configuring it for t
 
 Most instrumentations can be configured using the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. Ex: `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318`.
 
-If you're using a HyperDX provided SDK, you may need to give a non-empty `HYPERDX_API_KEY` as well, API keys are not validated in HyperDX Local and therefore can be any value
+If you're using a HyperDX provided SDK, you may need to give a non-empty `HYPERDX_API_KEY` as well, API keys are not validated in HyperDX Local and therefore can be any non-empty value.
+
+## Customizing Ports
+
+If you need to customize the app (8080) or api (8000) ports that HyperDX Local runs on, you'll need to modify the `docker run` command to forward the appropriate ports and set a few environment variables.
+
+Customizing the OpenTelemetry ports can simply be changed by modifying the port forwarding flags. Ex. Replacing `-p 4318:4318` with `-p 4999:4318` to change the OpenTelemetry HTTP port to 4999.
+
+```bash
+HYPERDX_APP_PORT=YOUR_CUSTOM_APP_PORT \
+HYPERDX_API_PORT=YOUR_CUSTOM_API_PORT; \
+docker run \
+-e HYPERDX_APP_PORT=$HYPERDX_APP_PORT \
+-e HYPERDX_API_PORT=$HYPERDX_API_PORT \
+-p $HYPERDX_API_PORT:8000 \
+-p 4318:4318 -p 4317:4317 \
+-p $HYPERDX_APP_PORT:8080 \
+-p 8002:8002 \
+sha256:a2954c091a3cf7e7de0263e6548bec075deb3259d305f5f2e09a7113a78c8e38
+```
 
 ## Notes
 
 ### Limitations vs Regular Open Source Version
 
 There are a few minor limitations compared to the regular open source version:
-- Single user mode only
+- Single user mode only (due to lack of auth requirement)
 - No support for management APIs
-- Alerts will not fire
+- No alerting support (alerts will not fire)
 - Log and DB query patterns will not be calculated
-- No persistence of data between restarts
+- No persistence of data if the container is torn down 
 
 ### Ports
 
@@ -56,9 +75,9 @@ There are a few minor limitations compared to the regular open source version:
 - `8002` - HyperDX HTTP Logging Endpoint
 - `8080` - HyperDX UI (Next.js)
 
-### Build Image
+### Build Image (Only For Development)
 
-To build the image, run the build script from the project root:
+To develop this image, you can build the image locally by running the build script from project root:
 
 ```
 ./docker/local/build.sh

docker/local/clickhouseConfig.xml

@@ -2,9 +2,9 @@
 <clickhouse>
     <logger>
         <level from_env="CLICKHOUSE_LOG_LEVEL" />
-        <console>true</console>
-        <log remove="remove" />
-        <errorlog remove="remove" />
+        <console>false</console>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
     </logger>
 
     <query_log>

docker/local/entry.sh

@@ -5,12 +5,15 @@ export HYPERDX_LOG_LEVEL="error"
 export CLICKHOUSE_LOG_LEVEL="error"
 
 # User-facing Services
-export AGGREGATOR_API_URL="http://localhost:8001"
 export INGESTOR_API_URL="http://localhost:8002"
-export SERVER_URL="http://localhost:8000"
-export FRONTEND_URL="http://localhost:8080"
+# User can specify either an entire SERVER_URL, or override slectively the 
+# HYPERDX_API_URL or HYPERDX_API_PORT from the defaults
+# Same applies to the frontend/app
+export SERVER_URL="${SERVER_URL:-${HYPERDX_API_URL:-http://localhost}:${HYPERDX_API_PORT:-8000}}"
+export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"
 
 # Internal Services
+export AGGREGATOR_API_URL="http://localhost:8001"
 export CLICKHOUSE_HOST="http://localhost:8123"
 export MONGO_URI="mongodb://localhost:27017/hyperdx"
 export REDIS_URI="redis://localhost:6379"
@@ -23,24 +26,33 @@ export NEXT_TELEMETRY_DISABLED="1"
 echo "127.0.0.1      ingestor" >> /etc/hosts
 echo "127.0.0.1      aggregator" >> /etc/hosts
 
+echo "Visit the HyperDX UI at $FRONTEND_URL/search"
+echo ""
+echo "Send OpenTelemetry data via"
+echo "http/protobuf: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318"
+echo "gRPC: OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317"
+echo ""
+echo ""
+
 # Start Clickhouse Server
 /entrypoint.sh &
 
 # Start Redis Server
-redis-server &
+redis-server > /var/log/redis.log 2>&1 &
 
 # Start Mongo Server
-mongod --quiet --dbpath /data/db &
+mongod --quiet --dbpath /data/db > /var/log/mongod.log 2>&1 &
 
 # Start Vector Ingestor
 ENABLE_GO_PARSER="false" \
 GO_PARSER_API_URL="http://go-parser:7777" \
 ENABLE_TOKEN_MATCHING_CHECK="false" \
 vector \
+  -qq \
   -c /etc/vector/sources.toml \
   -c /etc/vector/core.toml \
   -c /etc/vector/http-sinks.toml \
-  --require-healthy true &
+  --require-healthy true > /var/log/vector.log 2>&1 &
 
 # Start Otel Collector
 otelcol-contrib --config /etc/otelcol-contrib/config.yaml &
@@ -50,19 +62,20 @@ APP_TYPE=aggregator \
 CLICKHOUSE_USER=aggregator \
 CLICKHOUSE_PASSWORD=aggregator \
 PORT=8001 \
-node /app/api/build/index.js &
+node /app/api/build/index.js > /var/log/aggregator.log 2>&1 &
 
 # Api
 APP_TYPE=api \
 CLICKHOUSE_USER=api \
 CLICKHOUSE_PASSWORD=api \
 PORT=8000 \
-node /app/api/build/index.js &
+node /app/api/build/index.js > /var/log/api.log 2>&1 &
 
 # App
 NODE_ENV=production \
 PORT=8080 \
-node /app/app/server.js &
+NEXT_PUBLIC_SERVER_URL="${SERVER_URL}" \
+node /app/app/server.js > /var/log/app.log 2>&1 &
 
 # Wait for any process to exit
 wait -n

packages/app/pages/_app.tsx

@@ -14,6 +14,7 @@ import {
   MantineThemeOverride,
 } from '@mantine/core';
 
+import { apiConfigs } from '../src/api';
 import * as config from '../src/config';
 import { useConfirmModal } from '../src/useConfirm';
 import { QueryParamProvider as HDXQueryParamProvider } from '../src/useQueryParam';
@@ -125,13 +126,23 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) {
     fetch('/api/config')
       .then(res => res.json())
       .then(_jsonData => {
+        // Set API url dynamically for users who aren't rebuilding
+        try {
+          const url = new URL(_jsonData.apiServerUrl);
+          if (url != null) {
+            apiConfigs.prefixUrl = url.toString().replace(/\/$/, '');
+          }
+        } catch (err) {
+          // ignore
+        }
+
         if (_jsonData?.apiKey) {
           let hostname;
           try {
             const url = new URL(_jsonData.apiServerUrl);
             hostname = url.hostname;
           } catch (err) {
-            // console.log(err);
+            // ignore
           }
           HyperDX.init({
             apiKey: _jsonData.apiKey,
@@ -149,7 +160,7 @@ export default function MyApp({ Component, pageProps }: AppPropsWithLayout) {
       .catch(err => {
         // ignore
       });
-  });
+  }, []);
 
   const getLayout = Component.getLayout ?? (page => page);
 

packages/app/pages/api/config.ts

@@ -1,10 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 
 import {
-  API_SERVER_URL,
   HDX_API_KEY,
   HDX_COLLECTOR_URL,
   HDX_SERVICE_NAME,
+  SERVER_URL,
 } from '../../src/config';
 
 type ResponseData = {
@@ -20,7 +20,7 @@ export default function handler(
 ) {
   res.status(200).json({
     apiKey: HDX_API_KEY,
-    apiServerUrl: API_SERVER_URL,
+    apiServerUrl: SERVER_URL,
     collectorUrl: HDX_COLLECTOR_URL,
     serviceName: HDX_SERVICE_NAME,
   });

packages/app/src/AppNav.tsx

@@ -27,8 +27,9 @@ import { version } from '../package.json';
 import api from './api';
 import AuthLoadingBlocker from './AuthLoadingBlocker';
 import {
-  API_SERVER_URL,
+  IS_LOCAL_MODE,
   K8S_DASHBOARD_ENABLED,
+  SERVER_URL,
   SERVICE_DASHBOARD_ENABLED,
 } from './config';
 import Icon from './Icon';
@@ -1370,14 +1371,16 @@ export default function AppNav({ fixed = false }: { fixed?: boolean }) {
                   </span>
                 </Link>
               </div>
-              <div className="my-4">
-                <Link href={`${API_SERVER_URL}/logout`} legacyBehavior>
-                  <span role="button" className="text-muted-hover">
-                    <i className="bi bi-box-arrow-left text-slate-300" />{' '}
-                    {!isCollapsed && <span>Logout</span>}
-                  </span>
-                </Link>
-              </div>
+              {!IS_LOCAL_MODE && (
+                <div className="my-4">
+                  <Link href={`${SERVER_URL}/logout`} legacyBehavior>
+                    <span role="button" className="text-muted-hover">
+                      <i className="bi bi-box-arrow-left text-slate-300" />{' '}
+                      {!isCollapsed && <span>Logout</span>}
+                    </span>
+                  </Link>
+                </div>
+              )}
               <div className="d-flex justify-content-end align-items-end">
                 <span className="text-muted-hover fs-8.5">v{version}</span>
               </div>

packages/app/src/AuthPage.tsx

@@ -14,7 +14,7 @@ import {
 } from '@mantine/core';
 
 import api from './api';
-import { API_SERVER_URL } from './config';
+import { SERVER_URL } from './config';
 import * as config from './config';
 import LandingHeader from './LandingHeader';
 import { CheckOrX, PasswordCheck } from './PasswordCheck';
@@ -26,6 +26,17 @@ type FormData = {
 };
 
 export default function AuthPage({ action }: { action: 'register' | 'login' }) {
+  const { data: team, isLoading: teamIsLoading } = api.useTeam();
+  const router = useRouter();
+
+  const isLoggedIn = Boolean(!teamIsLoading && team && !team.isDemo);
+
+  useEffect(() => {
+    if (isLoggedIn) {
+      router.push('/search');
+    }
+  }, [isLoggedIn, router]);
+
   const isRegister = action === 'register';
   const {
     register,
@@ -37,7 +48,6 @@ export default function AuthPage({ action }: { action: 'register' | 'login' }) {
     reValidateMode: 'onSubmit',
   });
 
-  const router = useRouter();
   const { err, msg } = router.query;
 
   const { data: installation } = api.useInstallation();
@@ -100,7 +110,7 @@ export default function AuthPage({ action }: { action: 'register' | 'login' }) {
       }
     : {
         controller: {
-          action: `${API_SERVER_URL}/login/password`,
+          action: `${SERVER_URL}/login/password`,
           method: 'POST',
         },
         email: { name: 'email' },

packages/app/src/JoinTeamPage.tsx

@@ -2,7 +2,7 @@ import { useRouter } from 'next/router';
 import { NextSeo } from 'next-seo';
 import { Button, Form } from 'react-bootstrap';
 
-import { API_SERVER_URL } from './config';
+import { SERVER_URL } from './config';
 
 export default function JoinTeam() {
   const router = useRouter();
@@ -23,7 +23,7 @@ export default function JoinTeam() {
             <div className="text-center">
               <Form
                 className="text-start"
-                action={`${API_SERVER_URL}/team/setup/${token}`}
+                action={`${SERVER_URL}/team/setup/${token}`}
                 method="POST"
               >
                 <Form.Label