From f4d676cc1eea296457bc38152bd6e59ac2b8d4c8 Mon Sep 17 00:00:00 2001 From: Alexandre Peixoto Ferreira Date: Sun, 13 Nov 2022 14:13:52 -0600 Subject: [PATCH 1/4] Add suport for k3s-edge as helm chart Signed-off-by: Alexandre Peixoto Ferreira --- charts/smarter-k3s-edge/Chart.yaml | 19 ++ charts/smarter-k3s-edge/README.md | 4 + charts/smarter-k3s-edge/ci/ci-values.yaml | 1 + charts/smarter-k3s-edge/default.conf | 92 ++++++++++ charts/smarter-k3s-edge/templates/common.yaml | 172 ++++++++++++++++++ .../smarter-k3s-edge/templates/service.yaml | 18 ++ charts/smarter-k3s-edge/values.yaml | 18 ++ 7 files changed, 324 insertions(+) create mode 100644 charts/smarter-k3s-edge/Chart.yaml create mode 100644 charts/smarter-k3s-edge/README.md create mode 100644 charts/smarter-k3s-edge/ci/ci-values.yaml create mode 100644 charts/smarter-k3s-edge/default.conf create mode 100644 charts/smarter-k3s-edge/templates/common.yaml create mode 100644 charts/smarter-k3s-edge/templates/service.yaml create mode 100644 charts/smarter-k3s-edge/values.yaml diff --git a/charts/smarter-k3s-edge/Chart.yaml b/charts/smarter-k3s-edge/Chart.yaml new file mode 100644 index 0000000..fffc8ec --- /dev/null +++ b/charts/smarter-k3s-edge/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: smarter-k3s-edge +version: 0.0.1 +appVersion: v1.25.3-k3s1 +description: K3s server on kubernetes +home: https://k3s.io/ +icon: https://k3s.io/img/k3s-logo-light.svg +keywords: +- k3s +- edge +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/maintainers: | + - name: Alexandre Ferreira + email: alexandref75@gmail.com + artifacthub.io/prerelease: "false" + artifacthub.io/signKey: | + fingerprint: 71EDA4E3D652DC73EB09E3A5387D298C169CF24E + url: https://smarter-project.github.io/documentation/pgp_keys.asc diff --git a/charts/smarter-k3s-edge/README.md b/charts/smarter-k3s-edge/README.md new file mode 100644 index 0000000..50a516f --- /dev/null +++ b/charts/smarter-k3s-edge/README.md @@ -0,0 +1,4 @@ +# k3s edge helm chart + +[k3s](https://k3s.io) is a lightweight kubernetes + diff --git a/charts/smarter-k3s-edge/ci/ci-values.yaml b/charts/smarter-k3s-edge/ci/ci-values.yaml new file mode 100644 index 0000000..8f3d5dd --- /dev/null +++ b/charts/smarter-k3s-edge/ci/ci-values.yaml @@ -0,0 +1 @@ +logLevel: debug diff --git a/charts/smarter-k3s-edge/default.conf b/charts/smarter-k3s-edge/default.conf new file mode 100644 index 0000000..aa38eb2 --- /dev/null +++ b/charts/smarter-k3s-edge/default.conf @@ -0,0 +1,92 @@ +user root root; + +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name server_domain_or_IP; + return 302 https://$server_name$request_uri; + } + server { + disable_symlinks off; + # SSL configuration + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; + ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; + # from https://cipherli.st/ + # and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4 valid=300s; + resolver_timeout 5s; + # Disable preloading HSTS for now. You can use the commented out header line that includes + # the "preload" directive if you understand the implications. + #add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"; + add_header Strict-Transport-Security "max-age=63072000; includeSubdomains"; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + ssl_dhparam /etc/ssl/certs/dhparam.pem; + root /var/www/html; + server_name _; + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + # proxy_pass http://localhost:8080; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + } + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php7.0-cgi alone: + # fastcgi_pass 127.0.0.1:9000; + # # With php7.0-fpm: + # fastcgi_pass unix:/run/php/php7.0-fpm.sock; + #} + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} + } +} diff --git a/charts/smarter-k3s-edge/templates/common.yaml b/charts/smarter-k3s-edge/templates/common.yaml new file mode 100644 index 0000000..c00cbc2 --- /dev/null +++ b/charts/smarter-k3s-edge/templates/common.yaml @@ -0,0 +1,172 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.application.appName }}-data +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.application.appName }} +data: + default.conf: | + user root root; + + worker_processes auto; + + error_log /var/log/nginx/error.log notice; + pid /var/run/nginx.pid; + + events { + worker_connections 1024; + } + + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name server_domain_or_IP; + return 302 https://$server_name$request_uri; + } + server { + disable_symlinks off; + # SSL configuration + listen {{ .Values.configuration.portHTTPS }} ssl http2 default_server; + listen [::]:{{ .Values.configuration.portHTTPS }} ssl http2 default_server; + ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; + ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; + # from https://cipherli.st/ + # and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4 valid=300s; + resolver_timeout 5s; + # Disable preloading HSTS for now. You can use the commented out header line that includes + # the "preload" directive if you understand the implications. + #add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"; + add_header Strict-Transport-Security "max-age=63072000; includeSubdomains"; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + ssl_dhparam /etc/ssl/certs/dhparam.pem; + root /var/www/html; + server_name _; + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + # proxy_pass http://localhost:8080; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + } + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php7.0-cgi alone: + # fastcgi_pass 127.0.0.1:9000; + # # With php7.0-fpm: + # fastcgi_pass unix:/run/php/php7.0-fpm.sock; + #} + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.application.appName }} + labels: + name: {{ .Values.application.appName }} +spec: + selector: + matchLabels: + name: {{ .Values.application.appName }} + template: + metadata: + labels: + name: {{ .Values.application.appName }} + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + spec: + containers: + - name: {{ .Values.application.appName }} + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + command: [ "/bin/k3s", + "server", + "--tls-san","{{ .Values.configuration.hostip }}", + "--advertise-address", "{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}", + "--https-listen-port", "{{ .Values.configuration.port }}", + "--disable-agent", + "--disable","traefik", + "--disable","metrics-server", + "--disable","coredns", + "--disable","local-storage", + "--flannel-backend=none" ] + volumeMounts: + - name: k3s-data + mountPath: /var/lib/rancher/k3s + - name: k3s-config + mountPath: /etc/rancher/k3s + ports: + - containerPort: {{ .Values.configuration.port }} + - name: {{ .Values.application.appName }}-nginx + image: nginx:1.23.2-alpine + command: [ "/bin/sh", + "-c", + "apk update;apk add openssl;echo -e '\n\n\n\n\n\n\n' | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt;openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048;mkdir -p /var/www/html;ln -s /etc/rancher/k3s/k3s.yaml /var/www/html/k3s.yaml.{{ .Values.configuration.id }};ln -s /var/lib/rancher/k3s/server/token /var/www/html/token.{{ .Values.configuration.id }};chmod -R ago+rw /var/www/html;nginx -c /etc/nginx/conf.d/default.conf -g 'daemon off;'" ] + volumeMounts: + - name: k3s-data + mountPath: /var/lib/rancher/k3s + - name: k3s-config + mountPath: /etc/rancher/k3s + - name: config + mountPath: /etc/nginx/conf.d + ports: + - containerPort: {{ .Values.configuration.portHTTPS }} + volumes: + - name: k3s-data + persistentVolumeClaim: + claimName: {{ .Values.application.appName }}-data + - name: k3s-config + emptyDir: {} + - name: config + configMap: + name: {{ .Values.application.appName }} diff --git a/charts/smarter-k3s-edge/templates/service.yaml b/charts/smarter-k3s-edge/templates/service.yaml new file mode 100644 index 0000000..f4a5c47 --- /dev/null +++ b/charts/smarter-k3s-edge/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.application.appName }} + labels: + name: {{ .Values.application.appName }} +spec: + selector: + name: {{ .Values.application.appName }} + ports: + - protocol: TCP + port: {{ .Values.configuration.port }} + name: {{ .Values.application.appName }} + - protocol: TCP + port: {{ .Values.configuration.portHTTPS }} + name: {{ .Values.application.appName }}-https + externalIPs: + - {{ .Values.configuration.hostIP }} diff --git a/charts/smarter-k3s-edge/values.yaml b/charts/smarter-k3s-edge/values.yaml new file mode 100644 index 0000000..3e81445 --- /dev/null +++ b/charts/smarter-k3s-edge/values.yaml @@ -0,0 +1,18 @@ +# + +application: + appName: smarter-k3s-server + +image: + repository: rancher/k3s + # @default -- chart.appVersion + tag: "" + pullPolicy: IfNotPresent + +configuration: + hostIP: 192.168.2.222 + # Use this in case of NATed AWS + #externalHostIP: 192.168.2.222 + port: 6443 + portHTTPS: 6453 + # set id to paqssword From 668be86b3e86a7f894fa6f66c260e952e17150a9 Mon Sep 17 00:00:00 2001 From: Alexandre Peixoto Ferreira Date: Tue, 15 Nov 2022 09:49:25 -0600 Subject: [PATCH 2/4] Make NGINX optional Signed-off-by: Alexandre Peixoto Ferreira --- charts/smarter-k3s-edge/templates/common.yaml | 8 ++++++++ charts/smarter-k3s-edge/templates/service.yaml | 2 ++ charts/smarter-k3s-edge/values.yaml | 1 + 3 files changed, 11 insertions(+) diff --git a/charts/smarter-k3s-edge/templates/common.yaml b/charts/smarter-k3s-edge/templates/common.yaml index c00cbc2..c3f791f 100644 --- a/charts/smarter-k3s-edge/templates/common.yaml +++ b/charts/smarter-k3s-edge/templates/common.yaml @@ -1,3 +1,4 @@ +{{- if .Values.configuration.portHTTPS }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -109,6 +110,7 @@ data: } } --- +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -140,13 +142,16 @@ spec: "--disable","coredns", "--disable","local-storage", "--flannel-backend=none" ] + {{- if .Values.configuration.portHTTPS }} volumeMounts: - name: k3s-data mountPath: /var/lib/rancher/k3s - name: k3s-config mountPath: /etc/rancher/k3s + {{- end }} ports: - containerPort: {{ .Values.configuration.port }} + {{- if .Values.configuration.portHTTPS }} - name: {{ .Values.application.appName }}-nginx image: nginx:1.23.2-alpine command: [ "/bin/sh", @@ -161,7 +166,9 @@ spec: mountPath: /etc/nginx/conf.d ports: - containerPort: {{ .Values.configuration.portHTTPS }} + {{- end }} volumes: + {{- if .Values.configuration.portHTTPS }} - name: k3s-data persistentVolumeClaim: claimName: {{ .Values.application.appName }}-data @@ -170,3 +177,4 @@ spec: - name: config configMap: name: {{ .Values.application.appName }} + {{- end }} diff --git a/charts/smarter-k3s-edge/templates/service.yaml b/charts/smarter-k3s-edge/templates/service.yaml index f4a5c47..8094fbe 100644 --- a/charts/smarter-k3s-edge/templates/service.yaml +++ b/charts/smarter-k3s-edge/templates/service.yaml @@ -11,8 +11,10 @@ spec: - protocol: TCP port: {{ .Values.configuration.port }} name: {{ .Values.application.appName }} + {{- if .Values.configuration.portHTTPS }} - protocol: TCP port: {{ .Values.configuration.portHTTPS }} name: {{ .Values.application.appName }}-https + {{- end }} externalIPs: - {{ .Values.configuration.hostIP }} diff --git a/charts/smarter-k3s-edge/values.yaml b/charts/smarter-k3s-edge/values.yaml index 3e81445..916fb6a 100644 --- a/charts/smarter-k3s-edge/values.yaml +++ b/charts/smarter-k3s-edge/values.yaml @@ -14,5 +14,6 @@ configuration: # Use this in case of NATed AWS #externalHostIP: 192.168.2.222 port: 6443 + # Comment to remove NGINX portHTTPS: 6453 # set id to paqssword From 1ef55c326eec6ad0a2a18d34b4e642a3a5f783bf Mon Sep 17 00:00:00 2001 From: Alexandre Peixoto Ferreira Date: Tue, 15 Nov 2022 11:37:34 -0600 Subject: [PATCH 3/4] Add k3s-start.sh script to NGINX Signed-off-by: Alexandre Peixoto Ferreira --- charts/smarter-k3s-edge/templates/common.yaml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/charts/smarter-k3s-edge/templates/common.yaml b/charts/smarter-k3s-edge/templates/common.yaml index c3f791f..dbd34c7 100644 --- a/charts/smarter-k3s-edge/templates/common.yaml +++ b/charts/smarter-k3s-edge/templates/common.yaml @@ -109,6 +109,24 @@ data: #} } } + k3s-start.sh: | + #!/bin/bash + # + curl -sflkO https://{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}:{{ .Values.configuration.portHTTPS }}/token.{{ .Values.configuration.id }} + curl -sflkO https://{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}:{{ .Values.configuration.portHTTPS }}/k3s.yaml.{{ .Values.configuration.id }} + export INSTALL_K3S_VERSION=$(echo "{{ default .Chart.AppVersion .Values.image.tag }}" | sed -e "s/-k3/+k3/") + export K3S_TOKEN=$(cat token.{{ .Values.configuration.id }}) + export K3S_URL=$(grep server: k3s.yaml.{{ .Values.configuration.id }} | sed -e "s/^ *.server: *//") + + curl -sfL https://get.k3s.io | \\ + sh -s - \\ + --kubelet-arg cluster-dns=169.254.0.2 \\ + --log /var/log/k3s.log \\ + --node-label smarter.nodetype=unknown \\ + --node-label smarter.nodemodel=unknown \\ + --node-label smarter.type=edge \\ + --node-taint smarter.type=edge:NoSchedule \\ + --node-label smarter-build=user-installed --- {{- end }} apiVersion: apps/v1 @@ -156,7 +174,7 @@ spec: image: nginx:1.23.2-alpine command: [ "/bin/sh", "-c", - "apk update;apk add openssl;echo -e '\n\n\n\n\n\n\n' | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt;openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048;mkdir -p /var/www/html;ln -s /etc/rancher/k3s/k3s.yaml /var/www/html/k3s.yaml.{{ .Values.configuration.id }};ln -s /var/lib/rancher/k3s/server/token /var/www/html/token.{{ .Values.configuration.id }};chmod -R ago+rw /var/www/html;nginx -c /etc/nginx/conf.d/default.conf -g 'daemon off;'" ] + "apk update;apk add openssl;echo -e '\n\n\n\n\n\n\n' | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt;openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048;mkdir -p /var/www/html;ln -s /etc/rancher/k3s/k3s.yaml /var/www/html/k3s.yaml.{{ .Values.configuration.id }};ln -s /var/lib/rancher/k3s/server/token /var/www/html/token.{{ .Values.configuration.id }};ln -s /etc/nginx/conf.d/k3s-start.sh /var/www/html/k3s-start.sh.{{ .Values.configuration.id }};chmod -R ago+rw /var/www/html;nginx -c /etc/nginx/conf.d/default.conf -g 'daemon off;'" ] volumeMounts: - name: k3s-data mountPath: /var/lib/rancher/k3s From 3d3c20f5a1808055f2b1396d8dd814f4660159c0 Mon Sep 17 00:00:00 2001 From: Alexandre Peixoto Ferreira Date: Tue, 15 Nov 2022 13:04:07 -0600 Subject: [PATCH 4/4] Adding documentation of smarter-k3s-edge Signed-off-by: Alexandre Peixoto Ferreira --- README.md | 6 +++--- k3s-edge-server.md | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9a3bace..d2f5429 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ ## This demo makes the following assumptions about your environment In this guide we assume you have done the following: -- You should have a cloud-based k3s server dedicated for edge deployment (we will refer to this as k3s-edge-server) before proceeding any further - - if you don't have a k3s-edge-server, you can follow [these instructions](./k3s-edge-server.md) -- You should also have an installed InfluxDB and Grafana instance in a separate kubernetes cluster +- You should have an installed InfluxDB and Grafana instance in a separate kubernetes cluster (cloud or local). - these may be installed on a second cloud node, with its own k3s server, we will refer to this as the cloud-data-node - if you don't have a cloud-data-node, you can follow [these instructions](./k3s-cloud-server.md) +- You should have a cloud-based k3s server dedicated for edge deployment (we will refer to this as k3s-edge-server) before proceeding any further + - if you don't have a k3s-edge-server, you can follow [these instructions](./k3s-edge-server.md) - You will also need an installed k3s edge node which has already been setup to talk to k3s-edge-server - instructions for registering a node running a **64 bit kernel and user space** are available [here](./k3s-edge-server.md#Joining a k3s edge node to the cluster) diff --git a/k3s-edge-server.md b/k3s-edge-server.md index 0bb802f..f35b488 100644 --- a/k3s-edge-server.md +++ b/k3s-edge-server.md @@ -14,7 +14,7 @@ This document will help you run a Smarter k3s server * Storage: At least 10GB ### k3s edge server -* Local linux (x86_64 or arm64)/windows/MacOS machine with docker, AWS EC2 VM instance or Google Cloud Platform GCE VM instance +* The k3s edge server can be installed on Baremetal, docker or on a kubernetes cluster (AWS EKS, Google GCE, etc...). * Multiple k3s edge servers can be run in a single server if different server ports are used (HOSTPORT). ### dev machine @@ -31,7 +31,23 @@ This document will help you run a Smarter k3s server Make sure you open port 6443 or the port used in your instance installation in your firewall so external hosts can contact your new server. On AWS, you will need to do this by editing the security group policy and adding an inbound rule. -## Setting k3s server up +## Installing k3s + +### Kubernetes + +The helm chart [smarter-k3s-edge](./charts/smarter-k3s-edge) allows a k3s server to be installed in a Kubernetes cluster. Configuration ID should be a ssecure value (long enough to not be easy to guess). +```helm +helm install --set configuration.id=XXXXXX smarter-k3s-edge chart/smarter-k3s-edge +``` + +The k3s-install.sh script can be downloaded at the edge nodes by using the command: +```bash +curl -sflk https://:/k3s-start.sh. | sh +``` + +### Docker + +Setting k3s server up [k3s](https://github.com/k3s-io/k3s) repository and [Rancher docker hub](https://hub.docker.com/r/rancher/k3s/) provide docker images and artifacts (k3s) allowing k3s to run as container. This repository provides the file [k3s-start.sh](./scripts/k3s-start.sh) that automates that process and runs a k3s suitable to be a SMARTER k3s server