Add proxy start script (#972)

Signed-off-by: zty98751 <zty98751@alibaba-inc.com>
Co-authored-by: Kent Dong <ch3cho@qq.com>

helm/core/templates/configmap.yaml

@@ -3,7 +3,11 @@
     # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain
     trustDomain: "cluster.local"
     accessLogEncoding: TEXT
+    {{- if .Values.global.o11y.enabled }}
+    accessLogFile: "/var/log/proxy/access.log"
+    {{- else }}
     accessLogFile: "/dev/stdout"
+    {{- end }}
     ingressControllerMode: "OFF"
     accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"}
 

helm/core/templates/controller-deployment.yaml

@@ -70,6 +70,8 @@ spec:
             periodSeconds: 3
             timeoutSeconds: 5
           env:
+          - name: PILOT_ENABLE_HEADLESS_SERVICE_POD_LISTENERS
+            value: "false"
           - name: HIGRESS_SYSTEM_NS
             value: "{{ .Release.Namespace }}"
           - name: DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD

helm/core/templates/deployment.yaml

@@ -1,3 +1,4 @@
+{{- $o11y := .Values.global.o11y  }}
 {{- $unprivilegedPortSupported := true }}
 {{- range $index, $node := (lookup "v1" "Node" "default" "").items }}
     {{- $kernelVersion := $node.status.nodeInfo.kernelVersion }}
@@ -67,6 +68,40 @@ spec:
           value: "0"
       {{- end }}
       containers:
+        {{- if $o11y.enabled }}
+          {{- $config := $o11y.promtail }}
+        - name: promtail
+          image: {{ $config.image.repository }}:{{ $config.image.tag }}
+          imagePullPolicy: IfNotPresent
+          args:
+            - -config.file=/etc/promtail/promtail.yaml
+          env:
+            - name: 'HOSTNAME'
+              valueFrom:
+                fieldRef:
+                  fieldPath: 'spec.nodeName'
+          ports:
+            - containerPort: {{ $config.port }}
+              name: http-metrics
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /ready
+              port: {{ $config.port }}
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          volumeMounts:
+            - name: promtail-config
+              mountPath: "/etc/promtail"
+            - name: log
+              mountPath: /var/log/proxy
+            - name: tmp
+              mountPath: /tmp
+        {{- end }}
         - name: higress-gateway
           image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}"
           args:
@@ -88,7 +123,10 @@ spec:
               - ALL
             allowPrivilegeEscalation: false
             privileged: false
+          # When enabling lite metrics, the configuration template files need to be replaced.
+          {{- if not .Values.global.liteMetrics }}
             readOnlyRootFilesystem: true
+          {{- end }}
             runAsUser: 1337
             runAsGroup: 1337
             runAsNonRoot: true
@@ -102,7 +140,6 @@ spec:
             runAsGroup: 1337
             runAsNonRoot: false
             allowPrivilegeEscalation: true
-            readOnlyRootFilesystem: true
           {{- end }}
           env:
           - name: NODE_NAME
@@ -148,6 +185,10 @@ spec:
             value: "{{ $.Values.clusterName | default `Kubernetes` }}"
           - name: INSTANCE_NAME
             value: "higress-gateway"
+          {{- if .Values.global.liteMetrics }}
+          - name: LITE_METRICS
+            value: "on"
+          {{- end }}
           {{- if include "skywalking.enabled" . }}
           - name: ISTIO_BOOTSTRAP_OVERRIDE
             value: /etc/istio/custom-bootstrap/custom_bootstrap.json
@@ -212,6 +253,10 @@ spec:
           - mountPath: /opt/plugins
             name: local-wasmplugins-volume
           {{- end }}
+          {{- if $o11y.enabled }}
+          - mountPath: /var/log/proxy
+            name: log
+          {{- end }}
       {{- if .Values.gateway.hostNetwork }}
       hostNetwork: {{ .Values.gateway.hostNetwork }}
       dnsPolicy: ClusterFirstWithHostNet
@@ -258,6 +303,15 @@ spec:
         emptyDir: {}
       - name: proxy-socket
         emptyDir: {}
+      {{- if $o11y.enabled }}
+      - name: log
+        emptyDir: {}
+      - name: tmp
+        emptyDir: {}
+      - name: promtail-config
+        configMap:
+          name: higress-promtail
+      {{- end }}
       - name: podinfo
         downwardAPI:
           defaultMode: 420

helm/core/templates/promtail.yaml

@@ -0,0 +1,64 @@
+{{- $o11y := .Values.global.o11y  }}
+{{- if $o11y.enabled }}
+{{- $config := $o11y.promtail }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: higress-promtail
+  namespace: {{ .Release.Namespace }}
+data:
+  promtail.yaml: |
+    server:
+      log_level: info
+      http_listen_port: {{ $config.port }}
+
+    clients:
+    - url: http://higress-console-loki.{{ .Release.Namespace }}:3100/loki/api/v1/push
+
+    positions:
+      filename: /tmp/promtail-positions.yaml
+    target_config:
+      sync_period: 10s
+    scrape_configs:
+    - job_name: access-logs
+      static_configs:
+      - targets:
+        - localhost
+        labels:
+          __path__: /var/log/proxy/access.log
+      pipeline_stages:
+      - json:
+          expressions:
+            authority:
+            method:
+            path:
+            protocol:
+            request_id:
+            response_code:
+            response_flags:
+            route_name:
+            trace_id:
+            upstream_cluster:
+            upstream_host:
+            upstream_transport_failure_reason:
+            user_agent:
+            x_forwarded_for:
+      - labels:
+          authority:
+          method:
+          path:
+          protocol:
+          request_id:
+          response_code:
+          response_flags:
+          route_name:
+          trace_id:
+          upstream_cluster:
+          upstream_host:
+          upstream_transport_failure_reason:
+          user_agent:
+          x_forwarded_for:
+      - timestamp:
+          source: timestamp
+          format: RFC3339Nano
+{{- end }}

helm/core/values.yaml

@@ -1,5 +1,6 @@
 revision: ""
 global:
+  liteMetrics: false
   xdsMaxRecvMsgSize: "104857600"
   defaultUpstreamConcurrencyThreshold: 10000
   enableSRDS: true
@@ -337,6 +338,20 @@ global:
   # Use the Mesh Control Protocol (MCP) for configuring Istiod. Requires an MCP source.
   useMCP: false
 
+  # Observability (o11y) configurations
+  o11y:
+    enabled: false
+    promtail:
+      image:
+        repository: grafana/promtail
+        tag: 2.9.4
+      port: 3101
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+      securityContext: {}
+
   # The name of the CA for workload certificates.
   # For example, when caName=GkeWorkloadCertificate, GKE workload certificates
   # will be used as the certificates for workloads.