diff --git a/README.md b/README.md index 9a3bace..d2f5429 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ ## This demo makes the following assumptions about your environment In this guide we assume you have done the following: -- You should have a cloud-based k3s server dedicated for edge deployment (we will refer to this as k3s-edge-server) before proceeding any further - - if you don't have a k3s-edge-server, you can follow [these instructions](./k3s-edge-server.md) -- You should also have an installed InfluxDB and Grafana instance in a separate kubernetes cluster +- You should have an installed InfluxDB and Grafana instance in a separate kubernetes cluster (cloud or local). - these may be installed on a second cloud node, with its own k3s server, we will refer to this as the cloud-data-node - if you don't have a cloud-data-node, you can follow [these instructions](./k3s-cloud-server.md) +- You should have a cloud-based k3s server dedicated for edge deployment (we will refer to this as k3s-edge-server) before proceeding any further + - if you don't have a k3s-edge-server, you can follow [these instructions](./k3s-edge-server.md) - You will also need an installed k3s edge node which has already been setup to talk to k3s-edge-server - instructions for registering a node running a **64 bit kernel and user space** are available [here](./k3s-edge-server.md#Joining a k3s edge node to the cluster) diff --git a/charts/smarter-k3s-edge/Chart.yaml b/charts/smarter-k3s-edge/Chart.yaml new file mode 100644 index 0000000..fffc8ec --- /dev/null +++ b/charts/smarter-k3s-edge/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: smarter-k3s-edge +version: 0.0.1 +appVersion: v1.25.3-k3s1 +description: K3s server on kubernetes +home: https://k3s.io/ +icon: https://k3s.io/img/k3s-logo-light.svg +keywords: +- k3s +- edge +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/maintainers: | + - name: Alexandre Ferreira + email: alexandref75@gmail.com + artifacthub.io/prerelease: "false" + artifacthub.io/signKey: | + fingerprint: 71EDA4E3D652DC73EB09E3A5387D298C169CF24E + url: https://smarter-project.github.io/documentation/pgp_keys.asc diff --git a/charts/smarter-k3s-edge/README.md b/charts/smarter-k3s-edge/README.md new file mode 100644 index 0000000..50a516f --- /dev/null +++ b/charts/smarter-k3s-edge/README.md @@ -0,0 +1,4 @@ +# k3s edge helm chart + +[k3s](https://k3s.io) is a lightweight kubernetes + diff --git a/charts/smarter-k3s-edge/ci/ci-values.yaml b/charts/smarter-k3s-edge/ci/ci-values.yaml new file mode 100644 index 0000000..8f3d5dd --- /dev/null +++ b/charts/smarter-k3s-edge/ci/ci-values.yaml @@ -0,0 +1 @@ +logLevel: debug diff --git a/charts/smarter-k3s-edge/default.conf b/charts/smarter-k3s-edge/default.conf new file mode 100644 index 0000000..aa38eb2 --- /dev/null +++ b/charts/smarter-k3s-edge/default.conf @@ -0,0 +1,92 @@ +user root root; + +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name server_domain_or_IP; + return 302 https://$server_name$request_uri; + } + server { + disable_symlinks off; + # SSL configuration + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; + ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; + # from https://cipherli.st/ + # and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4 valid=300s; + resolver_timeout 5s; + # Disable preloading HSTS for now. You can use the commented out header line that includes + # the "preload" directive if you understand the implications. + #add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"; + add_header Strict-Transport-Security "max-age=63072000; includeSubdomains"; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + ssl_dhparam /etc/ssl/certs/dhparam.pem; + root /var/www/html; + server_name _; + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + # proxy_pass http://localhost:8080; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + } + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php7.0-cgi alone: + # fastcgi_pass 127.0.0.1:9000; + # # With php7.0-fpm: + # fastcgi_pass unix:/run/php/php7.0-fpm.sock; + #} + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} + } +} diff --git a/charts/smarter-k3s-edge/templates/common.yaml b/charts/smarter-k3s-edge/templates/common.yaml new file mode 100644 index 0000000..dbd34c7 --- /dev/null +++ b/charts/smarter-k3s-edge/templates/common.yaml @@ -0,0 +1,198 @@ +{{- if .Values.configuration.portHTTPS }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.application.appName }}-data +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.application.appName }} +data: + default.conf: | + user root root; + + worker_processes auto; + + error_log /var/log/nginx/error.log notice; + pid /var/run/nginx.pid; + + events { + worker_connections 1024; + } + + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name server_domain_or_IP; + return 302 https://$server_name$request_uri; + } + server { + disable_symlinks off; + # SSL configuration + listen {{ .Values.configuration.portHTTPS }} ssl http2 default_server; + listen [::]:{{ .Values.configuration.portHTTPS }} ssl http2 default_server; + ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; + ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; + # from https://cipherli.st/ + # and https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; + ssl_ecdh_curve secp384r1; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4 valid=300s; + resolver_timeout 5s; + # Disable preloading HSTS for now. You can use the commented out header line that includes + # the "preload" directive if you understand the implications. + #add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"; + add_header Strict-Transport-Security "max-age=63072000; includeSubdomains"; + add_header X-Frame-Options DENY; + add_header X-Content-Type-Options nosniff; + ssl_dhparam /etc/ssl/certs/dhparam.pem; + root /var/www/html; + server_name _; + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + # proxy_pass http://localhost:8080; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + } + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # include snippets/fastcgi-php.conf; + # + # # With php7.0-cgi alone: + # fastcgi_pass 127.0.0.1:9000; + # # With php7.0-fpm: + # fastcgi_pass unix:/run/php/php7.0-fpm.sock; + #} + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} + } + } + k3s-start.sh: | + #!/bin/bash + # + curl -sflkO https://{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}:{{ .Values.configuration.portHTTPS }}/token.{{ .Values.configuration.id }} + curl -sflkO https://{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}:{{ .Values.configuration.portHTTPS }}/k3s.yaml.{{ .Values.configuration.id }} + export INSTALL_K3S_VERSION=$(echo "{{ default .Chart.AppVersion .Values.image.tag }}" | sed -e "s/-k3/+k3/") + export K3S_TOKEN=$(cat token.{{ .Values.configuration.id }}) + export K3S_URL=$(grep server: k3s.yaml.{{ .Values.configuration.id }} | sed -e "s/^ *.server: *//") + + curl -sfL https://get.k3s.io | \\ + sh -s - \\ + --kubelet-arg cluster-dns=169.254.0.2 \\ + --log /var/log/k3s.log \\ + --node-label smarter.nodetype=unknown \\ + --node-label smarter.nodemodel=unknown \\ + --node-label smarter.type=edge \\ + --node-taint smarter.type=edge:NoSchedule \\ + --node-label smarter-build=user-installed +--- +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.application.appName }} + labels: + name: {{ .Values.application.appName }} +spec: + selector: + matchLabels: + name: {{ .Values.application.appName }} + template: + metadata: + labels: + name: {{ .Values.application.appName }} + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + spec: + containers: + - name: {{ .Values.application.appName }} + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + command: [ "/bin/k3s", + "server", + "--tls-san","{{ .Values.configuration.hostip }}", + "--advertise-address", "{{ default .Values.configuration.hostIP .Values.configuration.externalHostIP}}", + "--https-listen-port", "{{ .Values.configuration.port }}", + "--disable-agent", + "--disable","traefik", + "--disable","metrics-server", + "--disable","coredns", + "--disable","local-storage", + "--flannel-backend=none" ] + {{- if .Values.configuration.portHTTPS }} + volumeMounts: + - name: k3s-data + mountPath: /var/lib/rancher/k3s + - name: k3s-config + mountPath: /etc/rancher/k3s + {{- end }} + ports: + - containerPort: {{ .Values.configuration.port }} + {{- if .Values.configuration.portHTTPS }} + - name: {{ .Values.application.appName }}-nginx + image: nginx:1.23.2-alpine + command: [ "/bin/sh", + "-c", + "apk update;apk add openssl;echo -e '\n\n\n\n\n\n\n' | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt;openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048;mkdir -p /var/www/html;ln -s /etc/rancher/k3s/k3s.yaml /var/www/html/k3s.yaml.{{ .Values.configuration.id }};ln -s /var/lib/rancher/k3s/server/token /var/www/html/token.{{ .Values.configuration.id }};ln -s /etc/nginx/conf.d/k3s-start.sh /var/www/html/k3s-start.sh.{{ .Values.configuration.id }};chmod -R ago+rw /var/www/html;nginx -c /etc/nginx/conf.d/default.conf -g 'daemon off;'" ] + volumeMounts: + - name: k3s-data + mountPath: /var/lib/rancher/k3s + - name: k3s-config + mountPath: /etc/rancher/k3s + - name: config + mountPath: /etc/nginx/conf.d + ports: + - containerPort: {{ .Values.configuration.portHTTPS }} + {{- end }} + volumes: + {{- if .Values.configuration.portHTTPS }} + - name: k3s-data + persistentVolumeClaim: + claimName: {{ .Values.application.appName }}-data + - name: k3s-config + emptyDir: {} + - name: config + configMap: + name: {{ .Values.application.appName }} + {{- end }} diff --git a/charts/smarter-k3s-edge/templates/service.yaml b/charts/smarter-k3s-edge/templates/service.yaml new file mode 100644 index 0000000..8094fbe --- /dev/null +++ b/charts/smarter-k3s-edge/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.application.appName }} + labels: + name: {{ .Values.application.appName }} +spec: + selector: + name: {{ .Values.application.appName }} + ports: + - protocol: TCP + port: {{ .Values.configuration.port }} + name: {{ .Values.application.appName }} + {{- if .Values.configuration.portHTTPS }} + - protocol: TCP + port: {{ .Values.configuration.portHTTPS }} + name: {{ .Values.application.appName }}-https + {{- end }} + externalIPs: + - {{ .Values.configuration.hostIP }} diff --git a/charts/smarter-k3s-edge/values.yaml b/charts/smarter-k3s-edge/values.yaml new file mode 100644 index 0000000..916fb6a --- /dev/null +++ b/charts/smarter-k3s-edge/values.yaml @@ -0,0 +1,19 @@ +# + +application: + appName: smarter-k3s-server + +image: + repository: rancher/k3s + # @default -- chart.appVersion + tag: "" + pullPolicy: IfNotPresent + +configuration: + hostIP: 192.168.2.222 + # Use this in case of NATed AWS + #externalHostIP: 192.168.2.222 + port: 6443 + # Comment to remove NGINX + portHTTPS: 6453 + # set id to paqssword diff --git a/k3s-edge-server.md b/k3s-edge-server.md index 0bb802f..f35b488 100644 --- a/k3s-edge-server.md +++ b/k3s-edge-server.md @@ -14,7 +14,7 @@ This document will help you run a Smarter k3s server * Storage: At least 10GB ### k3s edge server -* Local linux (x86_64 or arm64)/windows/MacOS machine with docker, AWS EC2 VM instance or Google Cloud Platform GCE VM instance +* The k3s edge server can be installed on Baremetal, docker or on a kubernetes cluster (AWS EKS, Google GCE, etc...). * Multiple k3s edge servers can be run in a single server if different server ports are used (HOSTPORT). ### dev machine @@ -31,7 +31,23 @@ This document will help you run a Smarter k3s server Make sure you open port 6443 or the port used in your instance installation in your firewall so external hosts can contact your new server. On AWS, you will need to do this by editing the security group policy and adding an inbound rule. -## Setting k3s server up +## Installing k3s + +### Kubernetes + +The helm chart [smarter-k3s-edge](./charts/smarter-k3s-edge) allows a k3s server to be installed in a Kubernetes cluster. Configuration ID should be a ssecure value (long enough to not be easy to guess). +```helm +helm install --set configuration.id=XXXXXX smarter-k3s-edge chart/smarter-k3s-edge +``` + +The k3s-install.sh script can be downloaded at the edge nodes by using the command: +```bash +curl -sflk https://:/k3s-start.sh. | sh +``` + +### Docker + +Setting k3s server up [k3s](https://github.com/k3s-io/k3s) repository and [Rancher docker hub](https://hub.docker.com/r/rancher/k3s/) provide docker images and artifacts (k3s) allowing k3s to run as container. This repository provides the file [k3s-start.sh](./scripts/k3s-start.sh) that automates that process and runs a k3s suitable to be a SMARTER k3s server