Merge remote-tracking branch 'origin/master' into auto-nuget-update/m…

…icrosoft.aspnet # Conflicts: # src/Microsoft.Health.CosmosDb/Microsoft.Health.CosmosDb.csproj # src/Microsoft.Health.Fhir.CosmosDb/Microsoft.Health.Fhir.CosmosDb.csproj # src/Microsoft.Health.Fhir.R4.Web/Microsoft.Health.Fhir.R4.Web.csproj # src/Microsoft.Health.Fhir.R5.Web/Microsoft.Health.Fhir.R5.Web.csproj # src/Microsoft.Health.Fhir.Stu3.Web/Microsoft.Health.Fhir.Stu3.Web.csproj
microsoft · Jul 10, 2020 · 569c592 · 569c592
2 parents 6dd743d + e3f489d
commit 569c592
Show file tree

Hide file tree

Showing 149 changed files with 2,578 additions and 5,479 deletions.
diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln
@@ -34,22 +34,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		testauthenvironment.json = testauthenvironment.json
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.CosmosDb", "src\Microsoft.Health.CosmosDb\Microsoft.Health.CosmosDb.csproj", "{AFE75E20-714A-444C-928D-490CB8469B58}"
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{B5F2D2DF-D0C7-4861-8259-F6A041DB9854}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "apps", "apps", "{80E8C5D3-E02C-49C0-969B-0C3A02FA1FF4}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SmartLauncher", "samples\apps\SmartLauncher\SmartLauncher.csproj", "{49CA3B9D-7A22-43E0-A434-B0EC38D119A5}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.CosmosDb.UnitTests", "src\Microsoft.Health.CosmosDb.UnitTests\Microsoft.Health.CosmosDb.UnitTests.csproj", "{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7}"
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FHIR", "FHIR", "{7457B218-2651-49B5-BED8-22233889516A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Stu3.Core", "src\Microsoft.Health.Fhir.Stu3.Core\Microsoft.Health.Fhir.Stu3.Core.csproj", "{C6759C03-6060-44D7-B44D-0BD89908B741}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Shared", "Shared", "{9DC49B6B-EBFC-44F6-B65F-8931078B630D}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.SqlServer", "src\Microsoft.Health.Fhir.SqlServer\Microsoft.Health.Fhir.SqlServer.csproj", "{87849B3F-5D12-41CA-A082-FAC065EF9FD8}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Extensions.Xunit", "src\Microsoft.Health.Extensions.Xunit\Microsoft.Health.Extensions.Xunit.csproj", "{4DDF9576-E22C-460A-937F-0EE0FEA6DB87}"
@@ -235,18 +229,10 @@ Global
 		{C015551C-0009-49CE-8C8D-B206BED0BB42}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C015551C-0009-49CE-8C8D-B206BED0BB42}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C015551C-0009-49CE-8C8D-B206BED0BB42}.Release|Any CPU.Build.0 = Release|Any CPU
-		{AFE75E20-714A-444C-928D-490CB8469B58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{AFE75E20-714A-444C-928D-490CB8469B58}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{AFE75E20-714A-444C-928D-490CB8469B58}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{AFE75E20-714A-444C-928D-490CB8469B58}.Release|Any CPU.Build.0 = Release|Any CPU
 		{49CA3B9D-7A22-43E0-A434-B0EC38D119A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{49CA3B9D-7A22-43E0-A434-B0EC38D119A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{49CA3B9D-7A22-43E0-A434-B0EC38D119A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{49CA3B9D-7A22-43E0-A434-B0EC38D119A5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C6759C03-6060-44D7-B44D-0BD89908B741}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C6759C03-6060-44D7-B44D-0BD89908B741}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C6759C03-6060-44D7-B44D-0BD89908B741}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -369,13 +355,10 @@ Global
 		{83F1D0BE-F8C4-4244-BB79-B77D92B8F092} = {7457B218-2651-49B5-BED8-22233889516A}
 		{2C4E1934-0AD7-4410-91F7-957F265D7C7B} = {7457B218-2651-49B5-BED8-22233889516A}
 		{C015551C-0009-49CE-8C8D-B206BED0BB42} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69}
-		{AFE75E20-714A-444C-928D-490CB8469B58} = {9DC49B6B-EBFC-44F6-B65F-8931078B630D}
 		{80E8C5D3-E02C-49C0-969B-0C3A02FA1FF4} = {B5F2D2DF-D0C7-4861-8259-F6A041DB9854}
 		{49CA3B9D-7A22-43E0-A434-B0EC38D119A5} = {80E8C5D3-E02C-49C0-969B-0C3A02FA1FF4}
-		{A6EDEC42-6828-4ADE-A0B4-2042B4AC65F7} = {9DC49B6B-EBFC-44F6-B65F-8931078B630D}
 		{7457B218-2651-49B5-BED8-22233889516A} = {8AD2A324-DAB5-4380-94A5-31F7D817C384}
 		{C6759C03-6060-44D7-B44D-0BD89908B741} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4}
-		{9DC49B6B-EBFC-44F6-B65F-8931078B630D} = {8AD2A324-DAB5-4380-94A5-31F7D817C384}
 		{87849B3F-5D12-41CA-A082-FAC065EF9FD8} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69}
 		{4DDF9576-E22C-460A-937F-0EE0FEA6DB87} = {B70945F4-01A6-4351-955B-C4A2943B5E3B}
 		{E02E5224-32CD-490F-B1E5-8509AD669334} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69}

diff --git a/THIRDPARTYNOTICES.md b/THIRDPARTYNOTICES.md
@@ -4,7 +4,7 @@ Do Not Translate or Localize
 
 This file is based on or incorporates material from the projects listed below (Third Party IP). The original copyright notice and the license under which Microsoft received such Third Party IP, are set forth below. Such licenses and notices are provided for informational purposes only. Microsoft licenses the Third Party IP to you under the licensing terms for the Microsoft product. Microsoft reserves all other rights not expressly granted under this agreement, whether by implication, estoppel or otherwise.
 
-## AngleSharp 0.13.0
+## AngleSharp 0.14.0
 * Component Source:   https://github.com/AngleSharp/AngleSharp/
 * Component Copyright and License:
   > The MIT License (MIT)
@@ -805,6 +805,80 @@ This file is based on or incorporates material from the projects listed below (T
   > (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
   > SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+## prometheus-net.AspNetCore 3.6.0
+* Component Source:   https://github.com/prometheus-net/prometheus-net
+* Component Copyright and License:
+  > The MIT License (MIT)
+  > 
+  > Copyright (c) 2015 andrasm
+  > 
+  > Permission is hereby granted, free of charge, to any person obtaining a copy
+  > of this software and associated documentation files (the "Software"), to deal
+  > in the Software without restriction, including without limitation the rights
+  > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  > copies of the Software, and to permit persons to whom the Software is
+  > furnished to do so, subject to the following conditions:
+  > 
+  > The above copyright notice and this permission notice shall be included in all
+  > copies or substantial portions of the Software.
+  > 
+  > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  > SOFTWARE.
+
+
+## prometheus-net.DotnetRuntime 3.3.1
+* Component Source:   https://github.com/djluck/prometheus-net.DotNetRuntime
+* Component Copyright and License:
+  > The MIT License (MIT)
+  > 
+  > Copyright (c) [year] [fullname]
+  > 
+  > Permission is hereby granted, free of charge, to any person obtaining a copy
+  > of this software and associated documentation files (the "Software"), to deal
+  > in the Software without restriction, including without limitation the rights
+  > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  > copies of the Software, and to permit persons to whom the Software is
+  > furnished to do so, subject to the following conditions:
+  > 
+  > The above copyright notice and this permission notice shall be included in all
+  > copies or substantial portions of the Software.
+  > 
+  > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  > SOFTWARE.
+
+## prometheus-net.SystemMetrics 1.0.1
+* Component Source:   https://github.com/Daniel15/prometheus-net.SystemMetrics
+* Component Copyright and License:
+  > Copyright (C) 2020 Daniel Lo Nigro (Daniel15)
+  > 
+  > Permission is hereby granted, free of charge, to any person obtaining a copy
+  > of this software and associated documentation files (the "Software"), to deal
+  > in the Software without restriction, including without limitation the rights
+  > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  > copies of the Software, and to permit persons to whom the Software is
+  > furnished to do so, subject to the following conditions:
+  > 
+  > The above copyright notice and this permission notice shall be included in all
+  > copies or substantial portions of the Software.
+  > 
+  > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  > SOFTWARE.
+
 ## StyleCop.Analyzers 1.1.118
 * Component Source:   https://github.com/DotNetAnalyzers/StyleCopAnalyzers/
 * Component Copyright and License:  

diff --git a/build/ci-pipeline.yml b/build/ci-pipeline.yml
@@ -143,6 +143,18 @@ stages:
       version: R5
       keyVaultName: $(DeploymentEnvironmentNameR5)
 
+- stage: DockerAddTag
+  displayName: 'Docker add master tag'
+  dependsOn:
+  - testStu3
+  - testR4
+  - testR5
+  jobs:
+  - template: ./jobs/docker-add-tag.yml
+    parameters:
+      sourceTag: $(ImageTag)
+      targetTag: 'master'
+
 - stage: cleanStorageAccounts
   displayName: 'Clean Storage Accounts'
   dependsOn: []

diff --git a/build/jobs/docker-add-tag.yml b/build/jobs/docker-add-tag.yml
@@ -0,0 +1,27 @@
+
+parameters:
+- name: sourceTag
+  type: string
+- name: targetTag
+  type: string
+
+jobs:
+- job: DockerAddTag
+  pool:
+    vmImage: 'ubuntu-latest'
+  steps:
+  - task: AzureCLI@2
+    displayName: 'Azure CLI: InlineScript'
+    inputs:
+      azureSubscription: $(ConnectedServiceName)
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        az acr login -n $(azureContainerRegistry)
+        for v in stu3 r4 r5; do
+          sourceImage="$(azureContainerRegistry)/${v}_fhir-server:${{parameters.sourceTag}}"
+          targetImage="$(azureContainerRegistry)/${v}_fhir-server:${{parameters.targetTag}}"
+          docker pull $sourceImage
+          docker tag $sourceImage $targetImage
+          docker push $targetImage
+        done
diff --git a/build/jobs/run-tests.yml b/build/jobs/run-tests.yml
@@ -39,6 +39,7 @@ jobs:
       testAssemblyVer2: |
         **\*${{ parameters.version }}.Tests.Integration*.dll
       searchFolder: '$(System.ArtifactsDirectory)'
+      testRunTitle: '${{ parameters.version }} Integration'
     env:
       'CosmosDb:Host': $(CosmosDb--Host)
       'CosmosDb:Key': $(CosmosDb--Key)
@@ -103,6 +104,7 @@ jobs:
       rerunType: 'basedOnTestFailurePercentage'
       rerunFailedThreshold: '95'
       rerunMaxAttempts: 3
+      testRunTitle: '${{ parameters.version }} CosmosDB'
 
 - job: 'sqlE2eTests'
   dependsOn: []
@@ -163,3 +165,4 @@ jobs:
       rerunType: 'basedOnTestFailurePercentage'
       rerunFailedThreshold: '95'
       rerunMaxAttempts: 3
+      testRunTitle: '${{ parameters.version }} SQL'
diff --git a/samples/kubernetes/README.md b/samples/kubernetes/README.md
@@ -214,3 +214,34 @@ ingress:
         hosts:
           - mytestfhir.example.com
 ```
+
+## Prometheus Metrics
+
+The FHIR service can (optionally) expose [Prometheus](https://prometheus.io) metrics on a seperate port. In order to collect (scrape) the metrics, will need to install Prometheus in your cluster. This can be done with the [Prometheus Operator](https://github.com/coreos/prometheus-operator).
+
+You can enable this Prometheus metrics with the `serviceMonitor.enabled` parameter by adding a `serviceMonitor` section to your values:
+
+```yaml
+serviceMonitor:
+  enabled: true
+  labels:
+    prometheus: monitor
+```
+
+The `label` has to match the `serviceMonitorSelector.matchLabels` for your `Prometheus` resource. You can find the match labels with:
+
+```bash
+kubectl get -n <prometheus namespace> Prometheus -o json | jq .items[0].spec.serviceMonitorSelector
+```
+
+which should have something like:
+
+```json
+{
+  "matchLabels": {
+    "prometheus": "monitor"
+  }
+}
+```
+
+to work with the settings above.
diff --git a/samples/kubernetes/deploy-aks.sh b/samples/kubernetes/deploy-aks.sh
@@ -209,11 +209,11 @@ export HELM_EXPERIMENTAL_OCI=1
 helm chart pull mcr.microsoft.com/k8s/asohelmchart:latest
 helm chart export mcr.microsoft.com/k8s/asohelmchart:latest --destination install-aso/
 
-helm upgrade --install aso ./install-aso/azure-service-operator \
+helm upgrade --install aso ./install-aso/azure-service-operator -n azureoperator-system --create-namespace  \
     --set azureSubscriptionID=$(echo $accountDetails | jq -r .id) \
     --set azureTenantID=$(echo $accountDetails | jq -r .tenantId) \
     --set azureClientID=$spId \
     --set azureClientSecret=$spPass \
     --set createNamespace=true \
-    --set image.repository="mcr.microsoft.com/k8s/azure-service-operator:latest"
+    --set image.repository="mcr.microsoft.com/k8s/azureserviceoperator:latest"
 
diff --git a/samples/kubernetes/helm/fhir-server/templates/deployment.yaml b/samples/kubernetes/helm/fhir-server/templates/deployment.yaml
@@ -32,6 +32,18 @@ spec:
           env:
             - name: ASPNETCORE_FORWARDEDHEADERS_ENABLED
               value: "true"
+            {{- if .Values.serviceMonitor.enabled }}
+            - name: PrometheusMetrics__enabled
+              value: "true"
+            - name: PrometheusMetrics__port
+              value: {{ .Values.serviceMonitor.port | quote }}
+            - name: PrometheusMetrics__dotnetRuntimeMetrics
+              value: {{ .Values.serviceMonitor.dotnetRuntimeMetrics | quote }}
+            - name: PrometheusMetrics__httpMetrics
+              value: {{ .Values.serviceMonitor.httpMetrics | quote }}
+            - name: PrometheusMetrics__systemMetrics
+              value: {{ .Values.serviceMonitor.systemMetrics | quote }}
+            {{- end }}
             {{- if .Values.appInsights.secretName }}
             - name: ApplicationInsights__InstrumentationKey
               valueFrom:
@@ -129,6 +141,9 @@ spec:
             - name: http
               containerPort: 8080
               protocol: TCP
+            - name: metrics
+              containerPort: {{ .Values.serviceMonitor.port }}
+              protocol: TCP
           livenessProbe:
             httpGet:
               path: /health/check

diff --git a/samples/kubernetes/helm/fhir-server/templates/metricsservice.yaml b/samples/kubernetes/helm/fhir-server/templates/metricsservice.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "fhir-server.fullname" . }}-metrics
+  labels:
+    {{- include "fhir-server.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.serviceMonitor.port }}
+      targetPort: {{ .Values.serviceMonitor.port }}
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "fhir-server.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/samples/kubernetes/helm/fhir-server/templates/servicemonitor.yaml b/samples/kubernetes/helm/fhir-server/templates/servicemonitor.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "fhir-server.fullname" . }}
+  labels:
+    {{- include "fhir-server.labels" . | nindent 4 }}
+    {{- with .Values.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+    - port: metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+  selector:
+    matchLabels:
+      {{- include "fhir-server.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/samples/kubernetes/helm/fhir-server/values.yaml b/samples/kubernetes/helm/fhir-server/values.yaml
@@ -86,6 +86,16 @@ appInsights:
   secretKey: "instrumentationKey"
 #  secretName: "cluster-app-insights"
 
+serviceMonitor:
+  enabled: false
+  port: 1234
+  labels: {}
+    # prometheus: monitor
+  interval: 30s
+  dotnetRuntimeMetrics: true
+  httpMetrics: true
+  systemMetrics: true
+
 security:
   enabled: false
   enableAadSmartOnFhirProxy: false

diff --git a/src/Microsoft.Health.CosmosDb.UnitTests/Microsoft.Health.CosmosDb.UnitTests.csproj b/src/Microsoft.Health.CosmosDb.UnitTests/Microsoft.Health.CosmosDb.UnitTests.csproj