diff --git a/README.md b/README.md index 61528d1..f82f5a1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ______ ____ | | _____ _______|__| ______ # Overview -Polaris is an open-source, opiniated & validated architecture for hyper-scale enterprise clusters that allows for easy setup of a cluster with all the essentials ready for application development and deployment. The authors of Polaris believe that event-driven microservice architectures will eat the current legacy RESTful request/response world, and therefore a slant towards hyper-scale, streaming technology is evident in the Polaris design. +Polaris is an open-source, opiniated & validated architecture for hyper-scale enterprise clusters that allows for easy setup of a cluster with all the essentials ready for application development and deployment. The authors of Polaris believe that event-driven microservice architectures will eat the current legacy RESTful request/response world, and therefore a slant towards hyper-scale, streaming technology is evident in the Polaris design. Polaris has the following features: @@ -84,7 +84,7 @@ You can view the [kops aws docs](https://github.com/kubernetes/kops/blob/master/ ``` The Helm client can be installed either from source, or from pre-built binary releases. - + From Snap(Linux) $ sudo snap install helm --classic @@ -95,277 +95,41 @@ You can view the [kops aws docs](https://github.com/kubernetes/kops/blob/master/ $ choco install kubernetes-helm ``` -2. Generate the DEX Certificate Authority bits - -``` -$ dex/gen-dex-ca.sh -``` - -3. Create SSH key that will be used by the Kubernetes cluster +2. Run cluster script which creates your vanilla cluster it also generates yaml files in infrastructure/ ``` -$ ssh-keygen -t rsa -``` - -4. Create the cluster definition using kops - -``` -$ create-cluster.sh -``` - -5. Edit the cluster to CA cert, OIDC and additional policies: - -``` -$ kops edit cluster --state=s3://kops-state-bucket --name=example.cluster.k8s --yes - -Then add under spec: - fileAssets: - - name: dex-ca-cert - path: /srv/kubernetes/assets/dex-ca-cert.pem - roles: [Master] # a list of roles to apply the asset to, zero defaults to all - content: | - *<< CONTENTS OF /dex/ca/dex-ca-cert.pem >>* - kubeAPIServer: - oidcIssuerURL: *<< URL FOR DEX HERE (eg. https://dex.example.cluster.k8s) >>* - oidcClientID: kubectl-access - oidcUsernameClaim: email - oidcGroupsClaim: groups - oidcCAFile: /srv/kubernetes/assets/dex-ca-cert.pem - additionalPolicies: - node: | - [ - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones", - "route53:ListResourceRecordSets" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ec2:AttachVolume", - "ec2:DetachVolume" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "codecommit:BatchGet*", - "codecommit:Get*", - "codecommit:Describe*", - "codecommit:List*", - "codecommit:GitPull" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "sqs:*", - "sns:*", - "cloudformation:*", - "ecr:*", - "dynamodb:*", - "s3:*" - ], - "Resource": "*" - } - ] -``` - -6. Edit polaris/values.yaml file to change the cluster name,region, etc. - -``` -Dex and Dex-k8s-authenticator: -- enables RBAC for kubectl accsess (logs user into their cluster), gets the CA - cert from s3 after the cluster has been created. - -Cluster-autoscaler: -- automatically adjusts the size of a Kubernetes Cluster so that all pods have a - place to run and there are no unneeded nodes. - -nginx-ingress: -- allows simple host or URL based HTTP routing. - -Flux: -- watches the changes on ECR and communicates updates to cluster to be - deployed - -polaris-prometheus-operator: - - Installs prometheus-operator (https://github.com/coreos/prometheusoperator) to create/configure/manage Prometheus clusters atop Kubernetes (i.e. - The Prometheus Operator for Kubernetes provides easy monitoring definitions - for Kubernetes services and deployment and management of Prometheus instances.) - -charts/polaris: - - installs the addons with the predefined configurations from the helm - packages located in the same directory and customized with all the values above +$ sudo ./run-cluster +Wait for cluster to come up it should have atleast one master node and one worker node in status READY also make sure all operators are READY ``` -7. Edit the node instance group to enable spot instances (Optional - for running cheap). - -``` -$ kops edit ig nodes --state=s3://kops-state-bucket --name=example.cluster.k8s +3. Run polaris script which install polaris operators on your cluster -Then add under spec: - maxPrice: "0.10" - minSize: 1 - maxSize: 6 ``` - -8. Create the cluster. - +$ sudo ./run-polaris ``` -Test run: -$ kops update cluster --state=s3://kops-state-bucket --name=example.cluster.k8s -Apply changes: -$ kops update cluster --state=s3://kops-state-bucket --name=example.cluster.k8s --yes +4. Install polaris-kafka -... wait for cluster to come up ... -$ watch -d 'kubectl get nodes -o wide; kubectl get pods --all-namespaces' ``` +$ helm --name polaris-kafka-cp-kafka --namespace app install polaris-kafka/ -9. Create Polaris Namespace and Install ServiceAccounts, helm and charts. - +Deploy other containers to this namespace to interact with Kafka topics ``` -$ kubectl create namespace polaris - -$ kubectl apply -f k8/serviceaccounts/tiller-serviceaccount.yaml - -$ helm init --service-account helm-tiller --upgrade --debug --wait -$ helm upgrade --namespace polaris --install polaris-prometheus-operator charts/prometheus-operator-0.0.29.tgz +5. Cleanup -$ helm upgrade --namespace polaris --install polaris charts/polaris ``` -10. Setup DEX - -``` -Install the dex certificates: - - - -$ kubectl create configmap dex-ca --namespace polaris --from-file dex-ca.pem=dex/ca/dex-ca-cert.pem - -$ kubectl create secret tls dex-ca --namespace polaris --cert=dex/ca/dex-ca-cert.pem --key=dex/ca/dex-ca-key.pem - -$ kubectl create secret tls dex-tls --namespace polaris --cert=dex/ca/dex-issuer-cert.pem --key=dex/ca/dex-issuer-key.pem - -Hit dex on https://dex.example.cluster.k8s/.well-known/openid-configuration and ensure you get the dex-kube-issuer cert. - -Modify charts/dex-k8s-authenticator/values.yaml and ensure: -1. CA certificate link is set to public in S3 -2. CA certificate contents exists in cacerts section (as base64 encoded value) - -Install a clusterrole for the admin@example.com administrator: - -$ kubectl apply --namespace polaris -f k8/serviceaccounts/admin@example.com.yaml -``` - -11. Login and get a kubectl token: - -``` -https://login.example.cluster.k8s - -Load up the kube-config as directed (maybe take a backup of existing!) -``` - -12. Setup Flux for CD - -``` -Create a code-commit repo in AWS (manually for now...) - e.g. kubernetes-example-cluster. - -Create an IAM user in AWS (manually for now...) - e.g. flux-example-user. - -Create an HTTPS Git credentials for AWS CodeCommit for that IAM user, and note the -username and password. - -Edit charts/flux/values.yaml and ensure you setup the following: -git.url to have the correct username and password, VERY IMPORTANT that the password is URLEncoded! Otherwise you will get weird errors from flux. - -$ kubectl create namespace devops - -$ helm upgrade --namespace devops --install flux k8/charts/flux - -Watch flux log itself connecting and syncing the repository. - -You should now be able to: - -$ fluxctl --k8s-fwd-ns polaris list-controllers - -$ fluxctl --k8s-fwd-ns polaris list-images - -Any specs you put in /cluster-repo and push will be applied to the cluster. - -Charts must be in /cluster-repo/charts and a corresponding release/blah.yaml containing -a FluxHelmRelease for it would also be applied. - -Cool watch to see stuff happening: - -$ watch -d 'fluxctl --k8s-fwd-ns polaris -n example list-controllers; fluxctl --k8s-fwd-ns polaris -n example list-images -c example:deployment/example-example' - -Then to setup example as an automated deployment: - -$ fluxctl --k8s-fwd-ns polaris -n example automate -c example:fluxhelmrelease/example -``` - -13. Upgrade Cilium to newer version (to avoid a crash when applying CiliumNetworkPolicies): - -``` -$ kubectl edit deployment daemonset cilium -n kube-system - -Change from: - image: cilium/cilium:v1.0-stable -to: - image: cilium/cilium:v1.2 - -Then delete every cilium pod (and have it restart). -``` - -14. Install aws-service-operator (early beta, but cool for creating ECRs) - -``` -Edit values and make sure you have sane values: - -$ helm install --name=aws-service-operator k8/charts/aws-service-operator +$ kops delete cluster kops delete cluster example.cluster.k8s --state s3://{bucket_name} --yes -Test that it's working by pushing an ECRRepository into the flux pipe or manually applying it - then login to AWS and list. +Also remove files from dex folder and revert all changes in git to start again. ``` ## Other administrative stuff - Shell access to the cluster (using creators id_rsa): ``` -$ ssh -i ~/.ssh/id_rsa admin@api.example.cluster.k8s +$ ssh -i ~/.ssh/polaris@api.example.cluster.k8s ``` ## Related Polaris Projects diff --git a/charts/polaris/Chart.yaml b/charts/polaris/Chart.yaml index 7b5cacd..33326d5 100644 --- a/charts/polaris/Chart.yaml +++ b/charts/polaris/Chart.yaml @@ -1,7 +1,5 @@ apiVersion: v1 -name: Polaris appVersion: "1.0" -description: Polaris is an open-source, opinionated & validated architecture for hyper-scale AI first enterprise clusters. -sources: - - https://github.com/synthesis-labs/polaris +description: Polaris by Synthesis +name: polaris version: 1.0.0 diff --git a/charts/polaris/SOURCES.txt b/charts/polaris/SOURCES.txt index 5da3a17..a0c61dc 100644 --- a/charts/polaris/SOURCES.txt +++ b/charts/polaris/SOURCES.txt @@ -10,7 +10,5 @@ stable/external-dns stable/cluster-autoscaler stable/nginx-ingress stable/metrics-server -coreos/kube-prometheus -coreos/prometheus-operator stable/dex weaveworks/flux diff --git a/charts/polaris/charts/aws-service-operator/.helmignore b/charts/polaris/charts/aws-service-operator/.helmignore deleted file mode 100644 index f0c1319..0000000 --- a/charts/polaris/charts/aws-service-operator/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/polaris/charts/aws-service-operator/Chart.yaml b/charts/polaris/charts/aws-service-operator/Chart.yaml deleted file mode 100644 index 72df9a6..0000000 --- a/charts/polaris/charts/aws-service-operator/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for provisioning AWS resources using Kubernetes Custom Resources Definitions -name: aws-service-operator -version: 0.1.0 diff --git a/charts/polaris/charts/aws-service-operator/templates/NOTES.txt b/charts/polaris/charts/aws-service-operator/templates/NOTES.txt deleted file mode 100644 index 83b21de..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/NOTES.txt +++ /dev/null @@ -1,8 +0,0 @@ -Helm chart to manage AWS resources using Kubernetes Custom Resource Definitions. - -Example: Creating a CRD that will create a new ECR repository named example-repository: - -apiVersion: service-operator.aws/v1alpha1 -kind: ECRRepository -metadata: - name: example-repository diff --git a/charts/polaris/charts/aws-service-operator/templates/_helpers.tpl b/charts/polaris/charts/aws-service-operator/templates/_helpers.tpl deleted file mode 100644 index 35f3f17..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "aws-service-operator.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "aws-service-operator.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "aws-service-operator.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/polaris/charts/aws-service-operator/templates/clusterrole.yaml b/charts/polaris/charts/aws-service-operator/templates/clusterrole.yaml deleted file mode 100644 index a2c17fb..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/clusterrole.yaml +++ /dev/null @@ -1,48 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: aws-service-operator -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - - configmaps - - services - - events - verbs: - - get - - list - - watch - - create - - delete - - update - - patch -- apiGroups: - - extensions - resources: - - thirdpartyresources - verbs: - - get - - list - - watch - - create - - delete - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - service-operator.aws - resources: - - "*" - verbs: - - "*" diff --git a/charts/polaris/charts/aws-service-operator/templates/clusterrolebinding.yaml b/charts/polaris/charts/aws-service-operator/templates/clusterrolebinding.yaml deleted file mode 100644 index 4f29809..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: aws-service-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-service-operator -subjects: -- kind: ServiceAccount - name: aws-service-operator - namespace: {{ .Release.Namespace }} diff --git a/charts/polaris/charts/aws-service-operator/templates/deployment.yaml b/charts/polaris/charts/aws-service-operator/templates/deployment.yaml deleted file mode 100644 index 823c13e..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/deployment.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: {{ include "aws-service-operator.fullname" . }} - labels: - app: {{ include "aws-service-operator.name" . }} - chart: {{ include "aws-service-operator.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - template: - metadata: - annotations: - iam.amazonaws.com/role: "arn:aws:iam::{{ .Values.operator.account }}:role/aws-service-operator" - labels: - app: {{ include "aws-service-operator.name" . }} - release: {{ .Release.Name }} - spec: - serviceAccountName: aws-service-operator - containers: - - name: aws-service-operator - image: awsserviceoperator/aws-service-operator:v0.0.1-alpha2 - imagePullPolicy: Always - args: - - server - - --cluster-name={{ .Values.operator.cluster }} - - --region={{ .Values.operator.region }} - - --account-id={{ .Values.operator.account }} diff --git a/charts/polaris/charts/aws-service-operator/templates/serviceaccount.yaml b/charts/polaris/charts/aws-service-operator/templates/serviceaccount.yaml deleted file mode 100644 index 7373d9f..0000000 --- a/charts/polaris/charts/aws-service-operator/templates/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: aws-service-operator - #namespace: aws-service-operator diff --git a/charts/polaris/charts/aws-service-operator/values.yaml b/charts/polaris/charts/aws-service-operator/values.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/charts/polaris/charts/dex-k8s-authenticator/values.yaml b/charts/polaris/charts/dex-k8s-authenticator/values.yaml index e69de29..4fc3472 100644 --- a/charts/polaris/charts/dex-k8s-authenticator/values.yaml +++ b/charts/polaris/charts/dex-k8s-authenticator/values.yaml @@ -0,0 +1,100 @@ +# Default values for dex-k8s-authenticator. + +# Deploy environment label, e.g. dev, test, prod +global: + deployEnv: dev + +replicaCount: 1 + +image: + repository: mintel/dex-k8s-authenticator + tag: latest + pullPolicy: Always + +dexK8sAuthenticator: + port: 5555 + debug: false + web_path_prefix: / + #logoUrl: http:// + #tlsCert: /path/to/dex-client.crt + #tlsKey: /path/to/dex-client.key + clusters: + - name: <> + short_description: <> + description: "This cluster is build from a automated script" + client_secret: rkKR4TX1RsK8OvVVufe45KIsuxN4A86B + issuer: https://dex.<> + k8s_master_uri: https://api.<> + client_id: kubectl-access + redirect_uri: https://login.<>/callback/<> + # + # Make this particular link public in s3: + k8s_ca_uri: + <> + +service: + type: ClusterIP + port: 5555 + + # For nodeport, specify the following: + # type: NodePort + # nodePort: + +ingress: + enabled: true + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + hosts: + - login.<> + tls: + - hosts: + - login.<> + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +caCerts: + enabled: true + secrets: + # Array of Self Signed Certificates + # cat CA.crt | base64 -w 0 + # + # name: The internal k8s name of the secret we create. It's also used in + # the volumeMount name. It must respect the k8s naming convension (avoid + # upper-case and '.' to be safe). + # + # filename: The filename of the CA to be mounted. It must end in .crt for + # update-ca-certificates to work + # + # value: The base64 encoded value of the CA + # + #secrets: + #- name: ca-cert1 + # filename: ca1.crt + # value: LS0tLS1......X2F + #- name: ca-cert2 + # filename: ca2.crt + # value: DS1tFA1......X2F + # + # Get this value by: cat k8/dex/ca/dex-ca-cert.pem | base64 + - name: ca1 + filename: ca1.crt + value: + <> + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/charts/polaris/charts/dex/values.yaml b/charts/polaris/charts/dex/values.yaml index e69de29..f5473d1 100644 --- a/charts/polaris/charts/dex/values.yaml +++ b/charts/polaris/charts/dex/values.yaml @@ -0,0 +1,132 @@ +# Default values for dex +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +image: quay.io/dexidp/dex +imageTag: "v2.11.0" +imagePullPolicy: "IfNotPresent" + +inMiniKube: false + +nodeSelector: {} + +replicas: 1 + +# TBD - make this an ingress rather +service: + type: ClusterIP + port: 443 +# annotations: +# external-dns.alpha.kubernetes.io/hostname: dex.<> + # externalIPs: + +# Added these to create the ingress with correct hostname +# and class to match ingress-controller +ingress: + host: dex.<> + class: nginx + +resources: + # Normal resource usage of dex server + # limits: + # cpu: 100m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 50Mi + +extraVolumes: [] +extraVolumeMounts: [] + +selfSigned: + create: false + #image: gcr.io/google_containers/kubernetes-dashboard-init-amd64 + #imageTag: "v1.0.0" + #imagePullPolicy: "IfNotPresent" + #caDays: 10000 + #certDays: 10000 + #altNames: + #- dex.minikube.local + #altIPs: {} + #- 192.168.42.219 + +secret: {} +# Override the default secret names here. +# tlsName: dex-tls +# caName: dex-ca + +env: [] + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + +config: + issuer: https://dex.<> + storage: + type: kubernetes + config: + inCluster: true + logger: + level: debug + web: + https: 0.0.0.0:5556 + tlsCert: /etc/dex/tls/tls.crt + tlsKey: /etc/dex/tls/tls.key + connectors: [] +# - type: github +# id: github +# name: GitHub +# config: +# clientID: xxxxxxxxxxxxxxx +# clientSecret: yyyyyyyyyyyyyyyyyyyyy +# redirectURI: https://dex.minikube.local:5556/callback +# org: kubernetes + #- type: microsoft + #id: microsoft + ## Required field for connector name. + #name: Microsoft + #config: + ## Credentials can be string literals or pulled from the environment. + #clientID: 33d82f8b-4db4-4d99-b610-0768ddd246df + #clientSecret: lmwcHDOCUH74+zgbA152:]! + #redirectURI: https://dex.<>/callback + # tenant: organizations + # + # Also - edit the application manifest (in MS Apps) and + # edit to include the valid reply urls: + # "replyUrls": [ + # "https://dex.<>/callback" + #] + # + # Then - need to approve the access using something like this: + # + # https://login.microsoftonline.com/94cfdb85-3d23-4849-a066-5cdad965ccd8/adminconsent?client_id=33d82f8b-4db4-4d99-b610-0768ddd246df + + + oauth2: + skipApprovalScreen: true + + staticClients: + - id: kubectl-access + redirectURIs: + - 'http://127.0.0.1:5555/callback' + - 'https://login.<>/callback/<>' + name: 'Kubectl Access' + secret: rkKR4TX1RsK8OvVVufe45KIsuxN4A86B + + enablePasswordDB: true + staticPasswords: + - email: "admin@synthesis.com" + # bcrypt hash of the string "password" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" + username: "admin" + userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" diff --git a/charts/polaris/charts/kube-prometheus-0.0.105.tgz b/charts/polaris/charts/kube-prometheus-0.0.105.tgz deleted file mode 100644 index 9b0285e..0000000 Binary files a/charts/polaris/charts/kube-prometheus-0.0.105.tgz and /dev/null differ diff --git a/charts/polaris/values.yaml b/charts/polaris/values.yaml index b5e86a1..c3605d1 100644 --- a/charts/polaris/values.yaml +++ b/charts/polaris/values.yaml @@ -1,11 +1,10 @@ - global: - cluster-name: example.cluster.k8s + cluster-name: <> aws-service-operator: operator: - account: "123412341234" - cluster: example # Can't contain any weird characters! + account: "931486170612" + cluster: polaris # Can't contain any weird characters! region: eu-west-1 external-dns: @@ -22,7 +21,7 @@ cluster-autoscaler: cloudProvider: aws awsRegion: eu-west-1 autoscalingGroups: - - name: nodes.example.cluster.k8s + - name: nodes.<> minSize: 1 maxSize: 6 @@ -34,20 +33,18 @@ nginx-ingress: enabled: true extraArgs: enable-ssl-passthrough: true - proxy-read-timeout: "3600" - proxy-send-timeout: "3600" flux: helmOperator: create: true git: - chartsPath: "example/cluster-repo/charts" + chartsPath: "ref4/cluster-repo/charts" git: - url: "" # url for git repo + url: "git@github.com:davidwebstar34/ExampleFluxRepo.git" branch: "master" - path: "example/cluster-repo/" # directories to watch - user: "Flux on Example" - email: "flux@example.com" + path: "ref4/cluster-repo/" + user: "polaris" + email: "davidw4@synthesis.co.za" setAuthor: true pollInterval: "1m" registry: @@ -58,29 +55,42 @@ dex: tlsName: dex-tls caName: dex-ca ingress: - host: dex.example.cluster.k8s + host: dex.<> config: - issuer: https://dex.example.cluster.k8s - connectors: [] - staticClients: - - id: kubectl-access - redirectURIs: - - 'http://127.0.0.1:5555/callback' - - 'https://login.example.cluster.k8s/callback/example.cluster.k8s' - name: 'Kubectl Access' - secret: "" # set this and match it with dexK8sAuthenticator.clusters[0].client_secret - enablePasswordDB: true - staticPasswords: - - email: "admin@example.com" - # bcrypt hash of the string "password" - hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" - username: "admin" - userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" - - email: "retail@example.com" - # bcrypt hash of the string "password" - hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" - username: "retail" - userID: "08a8684b-db88-4b73-90a9-3cd1661f5467" + issuer: https://dex.<> + connectors: [] + #- type: microsoft + #id: microsoft + #name: Microsoft + #config: + #clientID: 33d82f8b-4db4-4d99-b610-0768ddd246df + #clientSecret: lmwcHDOCUH74+zgbA152:]! + #redirectURI: https://dex.<>/callback + # tenant: organizations + # + # Also - edit the application manifest (in MS Apps) and + # edit to include the valid reply urls: + # "replyUrls": [ + # "https://dex.ref4.onlabs.cloud/callback" + #] + # + # Then - need to approve the access using something like this: + # + # https://login.microsoftonline.com/94cfdb85-3d23-4849-a066-5cdad965ccd8/adminconsent?client_id=33d82f8b-4db4-4d99-b610-0768ddd246df + staticClients: + - id: kubectl-access + redirectURIs: + - 'http://127.0.0.1:5555/callback' + - 'https://login.<>/callback/<>' + name: 'Kubectl Access' + secret: rkKR4TX1RsK8OvVVufe45KIsuxN4A86B + enablePasswordDB: true + staticPasswords: + - email: "admin@synthesis.com" + # bcrypt hash of the string "password" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" + username: "admin" + userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" dex-k8s-authenticator: dexK8sAuthenticator: @@ -88,25 +98,26 @@ dex-k8s-authenticator: debug: false web_path_prefix: / clusters: - - name: example.cluster.k8s - short_description: "example Cluster" - description: "Example cluster" - client_secret: "" - issuer: https://dex.example.cluster.k8s - k8s_master_uri: https://api.example.cluster.k8s + - name: <> + short_description: <> + description: "This cluster is build from a automated script" + client_secret: rkKR4TX1RsK8OvVVufe45KIsuxN4A86B + issuer: https://dex.<> + k8s_master_uri: https://api.<> client_id: kubectl-access - redirect_uri: https://login.example.cluster.k8s/callback/example.cluster.k8s + redirect_uri: https://login.<>/callback/<> # # Make this particular link public in s3: k8s_ca_uri: - https://s3.eu-west-1.amazonaws.com/kops-state-bucket/example.cluster.k8s/pki/issued/ca/123412341234.crt # url to CA cert on s3 + <> caCerts: enabled: true secrets: - # Get this value by: cat dex/ca/dex-ca-cert.pem | base64 [-w 0] + # Get this value by: cat k8/dex/ca/dex-ca-cert.pem | base64 [-w 0] - name: ca1 filename: ca1.crt - value: "" + value: + <> ingress: enabled: true annotations: {} @@ -114,7 +125,7 @@ dex-k8s-authenticator: # kubernetes.io/tls-acme: "true" path: / hosts: - - login.example.cluster.k8s + - login.<> tls: - hosts: - - login.example.cluster.k8s + - login.<> diff --git a/charts/prometheus-operator-0.0.29.tgz b/charts/prometheus-operator-0.0.29.tgz deleted file mode 100644 index 81c3875..0000000 Binary files a/charts/prometheus-operator-0.0.29.tgz and /dev/null differ diff --git a/create-cluster.sh b/create-cluster.sh index 22ae9d9..48290ce 100755 --- a/create-cluster.sh +++ b/create-cluster.sh @@ -1,18 +1,17 @@ +#!/bin/bash +# A Simple Shell Script To Create a Polaris Cluster +# David Webstar - 16/02/2019 +export KUBECONFIG=/root/.kube/config +export AWS_REGION=${REGION} +export AWS_DEFAULT_REGION=${REGION} +export KOPS_STATE_STORE=s3://${STATE_BUCKET} +echo 'AWS_REGION: ' ${AWS_REGION} +echo 'KOPS_STATE_STORE: ' ${KOPS_STATE_STORE} -CLUSTER_NAME=example.cluster.k8s # name of the cluster -STATE_BUCKET="" # name of s3 bucket where the state of the cluster will live -REGION=eu-west-1 -AVAILABILITY_ZONES=${REGION}a,${REGION}b,${REGION}c # select AZ's - depending on the region but some may only have 2 -KUBERNETES_VERSION=1.10.5 -MASTER_ZONES=${REGION}b -CLUSTER_CIDR=10.0.0.0/20 - -# Newer kernel version for cilium support -IMAGE=595879546273/CoreOS-stable-1855.4.0-hvm - -MASTER_COUNT=1 -NODE_COUNT=3 +ssh-keygen -q -t rsa -f /root/.ssh/polaris -N '' +echo 'Generating polaris ssh key' +echo 'Creating vanilla cluster' # To allow overriding of etcd version (to v3) for cilium KOPS_FEATURE_FLAGS=SpecOverrideFlag \ kops create cluster \ @@ -20,7 +19,7 @@ kops create cluster \ --override=cluster.spec.etcdClusters[*].version=3.1.11 `# Specify etcd3 version (instead of etcd v2)` \ --state s3://${STATE_BUCKET} `# Name of bucket to store state` \ `#--dry-run # Dont actually do it` \ - `#--output yaml # Output of dry-run [yaml | json]` \ + `#--output yaml # Output of dry-run [yaml | json]` \ `#--out # Stdout redirect` \ `#--target # direct, terraform, cloudformation` \ `#--yes # Specify --yes to immediately create the cluster` \ @@ -48,15 +47,76 @@ kops create cluster \ --networking cilium `# kubenet, classic, external, kopeio-vxlan, kopeio), weave, flannel-vxlan, flannel, flannel-udp, calico, canal, kube-router, romana, amazon-vpc-routed-eni, cilium` \ --node-count ${NODE_COUNT} `# Number of worker nodes` \ `#--node-security-groups # Existing SGs to apply to nodes` \ - --node-size c4.xlarge `# Node instance type` \ + --node-size m4.xlarge `# Node instance type` \ `#--node-tenancy default # [default | dedicated]` \ --node-volume-size 30 `# Node volume size in GB` \ `#--project # Project to use (must be set on GCE)` \ `#--ssh-access # Restrict SSH access to this CIDR. If not set, access will not be restricted by IP. (default [0.0.0.0/0])` \ - `#--ssh-public-key # SSH public key to use (default "~/.ssh/id_rsa.pub")` \ + --ssh-public-key ~/.ssh/polaris.pub `# SSH public key to use (default "~/.ssh/id_rsa.pub")` \ `#--subnets # Set to use shared subnets` \ --topology private `# [public | private]` \ `#--utility-subnets # Set to use shared utility subnets` \ - `#--vpc # Set to use a shared VPC` \ + `#--vpc vpc-07d5a9727cf0103d7 # Set to use a shared VPC` \ --zones ${AVAILABILITY_ZONES} `# Zones in which to run the cluster (nodes?)` \ $1 $2 $3 $4 $5 $6 $7 $8 $9 + + +kops get --name ${CLUSTER_NAME} --state=s3://${STATE_BUCKET} -o yaml > infrastructure/vanilla_cluster.yaml + +ROOT=$( dirname "${BASH_SOURCE[0]}" ) +FOLDER=$ROOT/dex/ca + +echo Will create CA in $FOLDER + +mkdir -p $FOLDER +cd $FOLDER + +cat << EOF > req.cnf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = dex.${CLUSTER_NAME} +EOF + +# CA Private and Self-signed Certificate +# +openssl genrsa -out dex-ca-key.pem 2048 +openssl req -x509 -new -nodes -key dex-ca-key.pem -days 9999 -out dex-ca-cert.pem -subj "/CN=dex-kube-ca" + +# Issuer private key and signed by CA +# +openssl genrsa -out dex-issuer-key.pem 2048 +openssl req -new -key dex-issuer-key.pem -out dex-issuer-csr.pem -subj "/CN=dex-kube-issuer" -config req.cnf +openssl x509 -req -in dex-issuer-csr.pem -CA dex-ca-cert.pem -CAkey dex-ca-key.pem -CAcreateserial -out dex-issuer-cert.pem -days 9999 -extensions v3_req -extfile req.cnf + +cd - + + +DEX='https:\/\/dex.'${CLUSTER_NAME} + +sed 's/^/ /' dex/ca/dex-ca-cert.pem > dex/ca/temp.txt +sed -i -e '/<>/{r dex/ca/temp.txt' -e 'd' -e '}' permissions/permissions_scaffold +sed -i 's/<>/'${DEX}'/g' permissions/permissions_scaffold + +cat infrastructure/vanilla_cluster.yaml | awk ' +/api:/ { + line = $0; + while ((getline < "permissions/permissions_scaffold") > 0) {print}; + print line; + next +} +{print}' > infrastructure/dex_cluster.yaml + +kops replace -f infrastructure/dex_cluster.yaml --state ${KOPS_STATE_STORE} +kops update cluster ${CLUSTER_NAME} --state ${KOPS_STATE_STORE} --yes + +watch -d 'kubectl get nodes -o wide; kubectl get pods --all-namespaces' diff --git a/create-polaris.sh b/create-polaris.sh new file mode 100755 index 0000000..7a9eeeb --- /dev/null +++ b/create-polaris.sh @@ -0,0 +1,51 @@ +#!/bin/bash +export KUBECONFIG=/root/.kube/config +export AWS_REGION=${REGION} +export AWS_DEFAULT_REGION=${REGION} +export KOPS_STATE_STORE=s3://${STATE_BUCKET} +echo 'AWS_REGION: ' ${AWS_REGION} +echo 'KOPS_STATE_STORE: ' ${KOPS_STATE_STORE} + +# helm --tiller-connection-timeout 900 + +echo 'Getting certificate name from AWS' +TEMP_CRT_NAME=$(aws s3 ls ${KOPS_STATE_STORE}/${CLUSTER_NAME}/pki/issued/ca/ --recursive | awk '{print $4}' | grep -o [0-9]*.crt) +echo 'Certificate: '$TEMP_CRT_NAME +echo 'Making certificate public' +aws s3api put-object-acl --bucket kops-mimic --key ${CLUSTER_NAME}'/pki/issued/ca/'${TEMP_CRT_NAME} --acl public-read + + +TEMP_CRT_CREDS=$(cat dex/ca/dex-ca-cert.pem | base64 -w 0) +TEMP_CRT_NAME_ESCAPED=$(echo $TEMP_CRT_NAME | sed 's/\//\\\//g') + +sed -i 's/<>/'${CLUSTER_NAME}'/g' charts/polaris/values.yaml +sed -i 's/<>/'${CLUSTER_SHORT_DESCRIPTION}'/g' charts/polaris/values.yaml +sed -i 's/<>/'${TEMP_CRT_CREDS}'/g' charts/polaris/values.yaml +sed -i 's/<>/https:\/\/s3-eu-west-1.amazonaws.com\/'${STATE_BUCKET}'\/'${CLUSTER_NAME}'\/pki\/issued\/ca\/'${TEMP_CRT_NAME_ESCAPED}'/g' charts/polaris/values.yaml + +sed -i 's/<>/'${CLUSTER_NAME}'/g' charts/polaris/charts/dex/values.yaml +sed -i 's/<>/'${TEMP_CRT_CREDS}'/g' charts/polaris/charts/dex/values.yaml +sed -i 's/<>/https:\/\/s3-eu-west-1.amazonaws.com\/'${STATE_BUCKET}'\/'${CLUSTER_NAME}'\/pki\/issued\/ca\/'${TEMP_CRT_NAME_ESCAPED}'/g' charts/polaris/charts/dex/values.yaml + +sed -i 's/<>/'${CLUSTER_NAME}'/g' charts/polaris/charts/dex-k8s-authenticator/values.yaml +sed -i 's/<>/'${CLUSTER_SHORT_DESCRIPTION}'/g' charts/polaris/charts/dex-k8s-authenticator/values.yaml +sed -i 's/<>/'${TEMP_CRT_CREDS}'/g' charts/polaris/charts/dex-k8s-authenticator/values.yaml +sed -i 's/<>/https:\/\/s3-eu-west-1.amazonaws.com\/'${STATE_BUCKET}'\/'${CLUSTER_NAME}'\/pki\/issued\/ca\/'${TEMP_CRT_NAME_ESCAPED}'/g' charts/polaris/charts/dex-k8s-authenticator/values.yaml + +sed -i 's/<>/'${CLUSTER_NAME}'/g' ingress/ingress.yaml +sed -i 's/<>/'${CLUSTER_NAME}'/g' ingress/ingress.yaml + +kubectl create namespace polaris +kubectl apply -f serviceaccounts/tiller-serviceaccount.yaml +helm init --service-account helm-tiller --upgrade --debug --wait + +echo 'Installing helm charts this may take a few minutes' +helm upgrade --namespace polaris --install polaris charts/polaris + +kubectl create configmap dex-ca --namespace polaris --from-file dex-ca.pem=dex/ca/dex-ca-cert.pem +kubectl create secret tls dex-ca --namespace polaris --cert=dex/ca/dex-ca-cert.pem --key=dex/ca/dex-ca-key.pem +kubectl create secret tls dex-tls --namespace polaris --cert=dex/ca/dex-issuer-cert.pem --key=dex/ca/dex-issuer-key.pem + +kubectl apply -f ingress/ingress.yaml + +helm upgrade --namespace polaris --install polaris-prometheus-operator stable/prometheus-operator diff --git a/dex/README.md b/dex/README.md new file mode 100644 index 0000000..8d1c8b6 --- /dev/null +++ b/dex/README.md @@ -0,0 +1 @@ + diff --git a/dex/gen-dex-ca.sh b/dex/gen-dex-ca.sh deleted file mode 100755 index 782d7ae..0000000 --- a/dex/gen-dex-ca.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -ROOT=$( dirname "${BASH_SOURCE[0]}" ) -FOLDER=$ROOT/ca - -echo Will create CA in $FOLDER - -mkdir -p $FOLDER -cd $FOLDER - -cat << EOF > req.cnf -[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name - -[req_distinguished_name] - -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = dex.example.cluster.k8s -EOF - -# CA Private and Self-signed Certificate -# -openssl genrsa -out dex-ca-key.pem 2048 -openssl req -x509 -new -nodes -key dex-ca-key.pem -days 9999 -out dex-ca-cert.pem -subj "/CN=dex-kube-ca" - -# Issuer private key and signed by CA -# -openssl genrsa -out dex-issuer-key.pem 2048 -openssl req -new -key dex-issuer-key.pem -out dex-issuer-csr.pem -subj "/CN=dex-kube-issuer" -config req.cnf -openssl x509 -req -in dex-issuer-csr.pem -CA dex-ca-cert.pem -CAkey dex-ca-key.pem -CAcreateserial -out dex-issuer-cert.pem -days 9999 -extensions v3_req -extfile req.cnf - -cd - diff --git a/environment b/environment new file mode 100644 index 0000000..da2a955 --- /dev/null +++ b/environment @@ -0,0 +1,16 @@ +CLUSTER_NAME=cv.hackathon.s7s.cloud +CLUSTER_SHORT_DESCRIPTION=Computer-Vision +STATE_BUCKET=hackathon-computer-vision-bucket + +REGION=eu-west-1 +AVAILABILITY_ZONES=eu-west-1a +KUBERNETES_VERSION=1.10.5 + +MASTER_ZONES=eu-west-1a +CLUSTER_CIDR=10.192.0.0/16 +MASTER_COUNT=1 +NODE_COUNT=3 + +AWS_PROFILE=default + +IMAGE=595879546273/CoreOS-stable-1855.4.0-hvm diff --git a/infrastructure/README.md b/infrastructure/README.md new file mode 100644 index 0000000..8d1c8b6 --- /dev/null +++ b/infrastructure/README.md @@ -0,0 +1 @@ + diff --git a/ingress/ingress.yaml b/ingress/ingress.yaml new file mode 100644 index 0000000..cf7574e --- /dev/null +++ b/ingress/ingress.yaml @@ -0,0 +1,33 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: grafana + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - host: grafana.frodo.synthesis.rocks + http: + paths: + - path: / + backend: + serviceName: polaris-prometheus-operator-grafana + servicePort: 80 + +--- + +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: prometheus + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - host: prometheus.frodo.synthesis.rocks + http: + paths: + - path: / + backend: + serviceName: polaris-prometheus-operato-prometheus + servicePort: 9090 diff --git a/permissions/permissions_scaffold b/permissions/permissions_scaffold new file mode 100644 index 0000000..29e108d --- /dev/null +++ b/permissions/permissions_scaffold @@ -0,0 +1,80 @@ + fileAssets: + - name: dex-ca-cert + path: /srv/kubernetes/assets/dex-ca-cert.pem + roles: [Master] # a list of roles to apply the asset to, zero defaults to all + content: | + <> + kubeAPIServer: + oidcIssuerURL: <> + oidcClientID: kubectl-access + oidcUsernameClaim: email + oidcGroupsClaim: groups + oidcCAFile: /srv/kubernetes/assets/dex-ca-cert.pem + additionalPolicies: + node: | + [ + { + "Effect": "Allow", + "Action": [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ChangeResourceRecordSets" + ], + "Resource": [ + "arn:aws:route53:::hostedzone/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:DetachVolume" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "codecommit:BatchGet*", + "codecommit:Get*", + "codecommit:Describe*", + "codecommit:List*", + "codecommit:GitPull" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "sqs:*", + "sns:*", + "cloudformation:*", + "ecr:*", + "dynamodb:*", + "s3:*" + ], + "Resource": "*" + } + ] diff --git a/run-cluster.sh b/run-cluster.sh new file mode 100755 index 0000000..a84086d --- /dev/null +++ b/run-cluster.sh @@ -0,0 +1,12 @@ +mkdir -p "$HOME"/.ssh +mkdir -p "$HOME"/.aws +mkdir -p "$HOME"/.kube + +docker run --rm -it \ + -v "$HOME"/.ssh:/root/.ssh:rw \ + -v "$HOME"/.aws:/root/.aws:ro \ + -v "$HOME"/.kube:/root/.kube:rw \ + -v "$(pwd)":/workdir \ + -w /workdir \ + --env-file environment \ + webstar34/polaris:1.0.0 bash -c '/workdir/create-cluster.sh' diff --git a/run-polaris.sh b/run-polaris.sh new file mode 100755 index 0000000..9e38101 --- /dev/null +++ b/run-polaris.sh @@ -0,0 +1,8 @@ +docker run --rm -it \ + -v "$HOME"/.ssh:/root/.ssh:rw \ + -v "$HOME"/.aws:/root/.aws:ro \ + -v "$HOME"/.kube:/root/.kube:rw \ + -v "$(pwd)":/workdir \ + -w /workdir \ + --env-file environment \ + webstar34/polaris:1.0.0 bash -c '/workdir/create-polaris.sh'