Merge upstream

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Change Log
 
+## 3.2.0
+
+[Full Changelog](https://github.com/cloudfoundry/stratos/compare/3.1.0...3.2.0)
+
+This release contains a number of fixes and improvements:
+
+**Improvements:**
+
+- SSO_WHITELIST should be able to ignore path [\#4273](https://github.com/cloudfoundry/stratos/issues/4273)
+- Improve view that shows details for a metrics endpoint [\#4271](https://github.com/cloudfoundry/stratos/issues/4271)
+- Helm Chart: Add support for node selectors [\#4265](https://github.com/cloudfoundry/stratos/issues/4265)
+- Add documentation for list's 'max' feature, include info on 'fetch all' button [\#4259](https://github.com/cloudfoundry/stratos/issues/4259)
+- Backup Endpoints & Tokens [\#4228](https://github.com/cloudfoundry/stratos/issues/4228)
+
+**Fixes:**
+
+- Deployment time does not show correctly in diagnostics when deployed with Helm 3 [\#4261](https://github.com/cloudfoundry/stratos/issues/4261)
+- Pushing app from Stratos can sometimes fail due to expired token [\#4253](https://github.com/cloudfoundry/stratos/issues/4253)
+- Helm: Chart fails to render if `uaa` section is missing, docs misleading [\#4248](https://github.com/cloudfoundry/stratos/issues/4248)
+- Profile: Disabling polling fails to disable polling [\#4244](https://github.com/cloudfoundry/stratos/issues/4244)
+- App Summary: Github tab: Row highlight of deployed commit is obscured [\#4243](https://github.com/cloudfoundry/stratos/issues/4243)
+- Data Inaccuracies in PCF [\#4237](https://github.com/cloudfoundry/stratos/issues/4237)
+
 ## 3.1.0
 
 [Full Changelog](https://github.com/SUSE/stratos/compare/3.0.0...3.1.0)

build/tools/changelog.sh

@@ -68,9 +68,9 @@ function log() {
 }
 
 COMPARE_REPO=${REPO}
-QUERY="repo:${REPO}+milestone:3.1.0+state:closed"
+QUERY="repo:${REPO}+milestone:${MILESTONE}+state:closed"
 if [ -n "${FORK}" ]; then
-  FORK_QUERY="repo:${FORK}+milestone:3.1.0+state:closed"
+  FORK_QUERY="repo:${FORK}+milestone:${MILESTONE}+state:closed"
   COMPARE_REPO=${FORK}
 fi
 

deploy/ci/automation/e2e-clean-remnants.sh

@@ -120,6 +120,7 @@ USERS=$(cf curl "/v2/users?results-per-page=100" | jq -r .resources[].entity.use
 clean "$USERS" "-" "delete-user" "^(acceptance\.e2e\.travisci)(-remove-users)\.(20[0-9]*)[Tt]([0-9]*)[zZ].*"
 clean "$USERS" "-" "delete-user" "^(acceptance\.e2e\.travis)(-remove-users)\.(20[0-9]*)[Tt]([0-9]*)[zZ].*"
 clean "$USERS" "-" "delete-user" "^(acceptancee2etravis)(invite[0-9])(20[0-9]*)[Tt]([0-9]*)[zZ].*"
+clean "$USERS" "-" "delete-user" "^(acceptance\.e2e\.travisci)(-manage-by-username)\.(20[0-9]*)[Tt]([0-9]*)[zZ].*"
 
 # Routes
 echo "Cleaning routes"

deploy/ci/travis/helm-chart-unit-tests.sh

@@ -24,3 +24,15 @@ helm unittest console
 
 # Run lint
 helm lint console
+
+# Run helm3 lint as well
+echo "Installing Helm 3"
+export BINARY_NAME=helm3
+curl -fsSL -o get_helm3.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+chmod 700 get_helm3.sh
+./get_helm3.sh
+
+# RUn Helm 3 lint
+helm3 lint console
+
+echo "All done"

deploy/kubernetes/console/README.md

@@ -22,9 +22,11 @@ Check the repository was successfully added by searching for the `console`, for
 ```
 helm search console
 NAME               	CHART VERSION   APP VERSION	DESCRIPTION                                  
-stratos/console    	3.0.0           3.0.0      	A Helm chart for deploying Stratos UI Console
+stratos/console    	3.1.0           3.1.0      	A Helm chart for deploying Stratos UI Console
 ```
 
+> Note: Version numbers will depend on the version of Stratos available from the Helm repository
+
 > Note: Commands shown in this document are for Helm version 3. For Helm version 2, when installing, instead of supplying the name via the `--name` flag, it is supplied as the first argument, before the chart name.
 
 To install Stratos:
@@ -85,13 +87,11 @@ The following table lists the configurable parameters of the Stratos Helm chart
 |console.mariadb.host|Hostname of the database when using an external db||
 |console.mariadb.port|Port of the database when using an external db|3306|
 |console.mariadb.tls|TLS mode when connecting to database (true, false, skip-verify, preferred)|false|
-|console.uaa.protocol|Protocol to use when authenticating with the UAA|https://|
-|console.uaa.host|Host of the UAA to authenticate with ||
-|console.uaa.port|Port of the UAA to authenticate with ||
-|console.uaa.consoleClient|Client to use when authenticating with the UAA|cf|
-|console.uaa.consoleClientSecret|Client secret to use when authenticating with the UAA||
-|console.uaa.consoleAdminIdentifier|Scope that identifies an admin user of Stratos (e.g. cloud_controller.admin||
-|console.uaa.skipSSLValidation|Skip SSL validation when when authenticating with the UAA|false|
+|uaa.endpoint|URL of the UAA endpoint to authenticate with ||
+|uaa.consoleClient|Client to use when authenticating with the UAA|cf|
+|uaa.consoleClientSecret|Client secret to use when authenticating with the UAA||
+|uaa.consoleAdminIdentifier|Scope that identifies an admin user of Stratos (e.g. cloud_controller.admin||
+|uaa.skipSSLValidation|Skip SSL validation when when authenticating with the UAA|false|
 |env.SMTP_AUTH|Authenticate against the SMTP server using AUTH command when Sending User Invite emails|false|
 |env.SMTP_FROM_ADDRESS|From email address to use when Sending User Invite emails||
 |env.SMTP_USER|User name to use for authentication when Sending User Invite emails||
@@ -115,6 +115,9 @@ The following table lists the configurable parameters of the Stratos Helm chart
 |console.service.extraLabels|Additional labels to be added to all service resources||
 |console.service.ingress.annotations|Annotations to be added to the ingress resource||
 |console.service.ingress.extraLabels|Additional labels to be added to the ingress resource||
+|console.nodeSelector|Node selectors to use for the console Pod||
+|mariadb.nodeSelector|Node selectors to use for the database Pod||
+|configInit.nodeSelector|Node selectors to use for the configuration Pod||
 
 ## Accessing the Console
 
@@ -280,6 +283,7 @@ UAA configuration can be specified by providing the following configuration.
 Create a yaml file with the content below and and update according to your environment and save to a file called `uaa-config.yaml`.
 ```
 uaa:
+  url: https://uaa.cf-dev.io:2793
   protocol: https://
   port: 2793
   host: uaa.cf-dev.io
@@ -291,7 +295,6 @@ uaa:
 
 To install Stratos with the above specified configuration:
 
-
 ```
 kubectl create namespace console
 helm install my-console stratos/console --namespace=console -f uaa-config.yaml

deploy/kubernetes/console/templates/config-init.yaml

@@ -94,6 +94,12 @@ spec:
 {{ toYaml .Values.console.podExtraLabels | nindent 8 }}
 {{- end }}
     spec:
+{{- if .Values.configInit }}
+{{- if .Values.configInit.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.configInit.nodeSelector | trim  | indent 8 }}
+{{- end }}
+{{- end }}
       containers:
       - env:
         - name: "STRATOS_VOLUME_MIGRATION"
@@ -170,6 +176,10 @@ spec:
         app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
         helm.sh/chart:  "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
     spec:
+{{- if .Values.configInit }}
+      nodeSelector:
+{{ toYaml .Values.configInit.nodeSelector | indent 8 }}
+{{- end }}
       containers:
       - env:
         - name: "NAMESPACE"

deploy/kubernetes/console/templates/database.yaml

@@ -49,6 +49,12 @@ spec:
         {{ toYaml .Values.console.podExtraLabels | nindent 8 }}
         {{- end}}        
     spec:
+{{- if .Values.mariadb }}
+{{- if .Values.mariadb.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.mariadb.nodeSelector | trim | indent 8 }}
+{{- end }}
+{{- end }}
       containers:
       - name: mariadb
         image: {{.Values.kube.registry.hostname}}/{{.Values.kube.organization}}/{{.Values.images.mariadb}}:{{.Values.consoleVersion}}

deploy/kubernetes/console/templates/deployment.yaml

@@ -47,6 +47,12 @@ spec:
 {{ toYaml .Values.console.podExtraLabels | indent 8 }}
 {{- end }}
     spec:
+{{- if .Values.console }}
+{{- if .Values.console.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.console.nodeSelector | trim | indent 8 }}
+{{- end }}
+{{- end }}
       containers:
       - image: {{.Values.kube.registry.hostname}}/{{.Values.kube.organization}}/{{.Values.images.console}}:{{.Values.consoleVersion}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
@@ -184,7 +190,15 @@ spec:
           value: stratos.admin
         {{- else }}
         # UAA
-        {{- if or .Values.env.UAA_HOST .Values.env.DOMAIN }}  
+        # Highest priority setting to use is uaa.endpoint
+        {{- if .Values.uaa.endpoint }}
+        - name: UAA_ENDPOINT 
+          value: {{ .Values.uaa.endpoint | quote }}
+        - name: CONSOLE_ADMIN_SCOPE
+          value: {{ default "cloud_controller.admin" .Values.uaa.consoleAdminIdentifier }}
+        - name: SKIP_SSL_VALIDATION 
+          value: {{default "true" .Values.uaa.skipSSLValidation | quote}}
+        {{- else if or .Values.env.UAA_HOST .Values.env.DOMAIN }}          
         - name: UAA_ENDPOINT 
           value: {{ template "scfUaaEndpoint" . }}
         {{- if and .Values.env.DOMAIN (not .Values.console.autoRegisterCF) }}
@@ -219,7 +233,7 @@ spec:
         - name: HELM_CHART_VERSION
           value: "{{ .Chart.Version }}"
         - name: HELM_LAST_MODIFIED
-          value: "{{ .Release.Time }}"
+          value: "seconds:{{ now | unixEpoch }} nanos:0 "        
         - name: SSO_LOGIN
           value: {{default "false" .Values.console.ssoLogin | quote}}
         - name: SSO_OPTIONS

deploy/kubernetes/console/tests/config_init_test.yaml

@@ -0,0 +1,12 @@
+suite: test stratos configInit
+templates:
+  - config-init.yaml
+tests:
+  - it: should set kubernetes.io/arch if configInit.nodeSelector.kubernetes.io/arch is set
+    set:
+      configInit.nodeSelector.kubernetes.io/arch: amd64
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector.kubernetes.io/arch
+          value: amd64
+        documentIndex: 3

deploy/kubernetes/console/tests/database-test.yaml

@@ -0,0 +1,12 @@
+suite: test stratos database
+templates:
+  - database.yaml
+tests:
+  - it: should set kubernetes.io/arch if mariadb.nodeSelector.kubernetes.io/arch is set
+    set:
+      mariadb.nodeSelector.kubernetes.io/arch: amd64
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector.kubernetes.io/arch
+          value: amd64
+        documentIndex: 0

deploy/kubernetes/console/tests/deployment_test.yaml

@@ -67,3 +67,11 @@ tests:
           content:
             name: AUTO_REG_CF_URL
             value: https://autoreg.test.com
+  - it: should set kubernetes.io/arch if console.nodeSelector.kubernetes.io/arch is set
+    set:
+      console.nodeSelector.kubernetes.io/arch: amd64
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector.kubernetes.io/arch
+          value: amd64
+        documentIndex: 0

deploy/kubernetes/console/values.yaml

@@ -79,6 +79,7 @@ console:
   tlsSecretName:
 
   # URL of a Cloud Foundry to use for authentication and to auto-register on login
+  # Deprecated
   autoRegisterCF: ~
 
   # Custom annotations to apply to Stateful sets
@@ -104,6 +105,9 @@ console:
 
   # Extra labels to apply to Pods
   podExtraLabels: {}
+
+  # Node Selector for console Pod
+  nodeSelector: {}
 
 images:
   console: stratos-console
@@ -145,13 +149,24 @@ mariadb:
     accessMode: ReadWriteOnce
     size: 1Gi
     storageClass:
+  # Node selector for the database pod
+  nodeSelector: {}
+
+configInit:
+  # Node selector for the config init pod
+  nodeSelector: {}
+
+# UAA configuration
 uaa:
-  protocol: https://
-  port: 
-  host: 
+  # UAA endpoint (e.g. https://uaa.domain:2793)
+  endpoint: ~
+  # Client to use when authenticating (default is 'cf')
   consoleClient: 
+  # Client Secret to use when authenticating (default is '')
   consoleClientSecret: 
-  consoleAdminIdentifier: 
+  # Scope that determines if a user is a Stratos admin
+  consoleAdminIdentifier:
+  # Skip SSL validation when communicating with the UAA
   skipSSLValidation: false
 
 # SCF values compatability 

docs/cf-entity-scaling.md

@@ -0,0 +1,43 @@
+# Cloud Foundry Scaling - How Stratos Handles Large Lists
+
+Stratos presents collections of entities via the Stratos list component. The list component presents the collection in a paginated, sortable and searchable way in either a set of cards or a table.
+In order to achieve this, due to the limitations of the APIs used, the list may fetch all entities (as opposed to fetching entities for the visible page only) and paginate, sort and search locally.
+
+## Protecting Stratos from Large Collections
+In some cases the number of entities in a collection can be incredibly high. Stratos can decide to not fetch them all in order to protect the CF from
+a substantial number of requests. To do this the first page is fetched and the total number of entities is checked against a limit. If under
+the limit the remaining pages are asynchronously fetched. If over then the remaining pages are ignored and the user is informed that the list could not fetch all entities.
+
+Depending on the list, the user can then try to filter the collection such that the number of entities is below the limit. Depending on configuration
+the user also has the option to fetch all entities regardless of the limit.
+
+## Applicable Lists
+Currently, in 3.1.0, this large collection protection is only applicable to the following lists
+
+- Application Wall
+- Marketplace (Services)
+- Services (Service & User Provided Service Instances)
+- CF, Organisation and Space Users
+- CF Routes
+
+In the future we hope to expand this to all lists.
+
+## Determining the List Limit
+The limit at which we won't fetch all entities is determined, in least important to most important order, by
+
+1) The global CF default - 600
+2) The Jetstream override for all lists - by default not set
+
+In the future we hope to allow each end user of Stratos to determine their own limit (if they have the correct permissions).
+
+## Fetch All feature
+If the list hits the limit the user will be presented with a button to `Fetch All` entities. Clicking this button ensures the list reverts
+back as if there were no limit and thus fetching all entities. This feature is disabled by default and can be enabled by a Jetstream override.
+
+## Configuration
+The Jetstream overrides can be set via environment variable, or if in helm, as values.
+
+Environment Variable|Helm Value|Description|Default|
+|---|---|---|---|
+|`UI_LIST_MAX_SIZE`|`console.ui.listMaxSize`|Override the default maximum number of entities that a configured list can fetch. When a list meets this amount additional pages are not fetched||
+|`UI_LIST_ALLOW_LOAD_MAXED`|`console.ui.listAllowLoadMaxed`|If the maximum list size is met give the user the option to fetch all results|false|

docs/sso.md

@@ -55,11 +55,13 @@ uaac client update cf --authorized_grant_types authorization_code
 When SSO has been configured Stratos's log in request will contain a URL that tells SSO where to return to. When using a browser this is automatically populated. To avoid situations where this can be hijacked or called separately an SSO `state` whitelist can be provided via the environment variable `SSO_WHITELIST`. This is a comma separated list. For example...
 
 ```
-SSO_WHITELIST=https://your.domain
+SSO_WHITELIST=https://your.domain/*
 ```
 
 ```
-SSO_WHITELIST=https://your.domain,https://your.other.domain
+SSO_WHITELIST=https://your.domain/*,https://your.other.domain/*
 ```
 
 When set, any requests to log in with a different `state` will be denied.
+
+In order for the SSO `state` to match an entry from the whitelist the schema, hostname, port and path must match exactly. A wildcard `*` can be provided for the path to match anything.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stratos",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "Stratos Console",
   "main": "index.js",
   "scripts": {

src/frontend/packages/cf-autoscaler/src/features/autoscaler-metric-page/autoscaler-metric-page.component.spec.ts

@@ -2,18 +2,17 @@ import { DatePipe } from '@angular/common';
 import { async, ComponentFixture, TestBed } from '@angular/core/testing';
 import { NoopAnimationsModule } from '@angular/platform-browser/animations';
 import { RouterTestingModule } from '@angular/router/testing';
+import { createEmptyStoreModule } from '@stratos/store/testing';
 
 import { ApplicationService } from '../../../../cloud-foundry/src/features/applications/application.service';
 import { CoreModule } from '../../../../core/src/core/core.module';
 import { SharedModule } from '../../../../core/src/shared/shared.module';
 import { TabNavService } from '../../../../core/tab-nav.service';
 import { ApplicationServiceMock } from '../../../../core/test-framework/application-service-helper';
-import { createEmptyStoreModule } from '@stratos/store/testing';
 import { CfAutoscalerTestingModule } from '../../cf-autoscaler-testing.module';
 import { AutoscalerMetricPageComponent } from './autoscaler-metric-page.component';
 
-// TODO: Fix after metrics has been sorted - STRAT-152
-xdescribe('AutoscalerMetricPageComponent', () => {
+describe('AutoscalerMetricPageComponent', () => {
   let component: AutoscalerMetricPageComponent;
   let fixture: ComponentFixture<AutoscalerMetricPageComponent>;
 
@@ -47,10 +46,5 @@ xdescribe('AutoscalerMetricPageComponent', () => {
     expect(component).toBeTruthy();
   });
 
-  // TODO: Fix after metrics has been sorted - STRAT-152 (cause of `Cannot read property 'getEntityMonitor' of undefined` test failure)
-  it('Blocked', () => {
-    fail('Blocked: Requires metrics to be working (specifically metrics entities)');
-  });
-
   afterAll(() => { });
 });