Merge branch 'master' into network-test

Jenkinsfile

@@ -25,6 +25,7 @@ pipeline {
         ))
         timestamps()
         skipStagesAfterUnstable()
+        timeout(time: 5, unit: 'HOURS')
     }
 
     environment {

Jenkinsfile-E2E

@@ -21,6 +21,11 @@ pipeline {
         label 'skywalking'
     }
 
+    options {
+        timestamps()
+        timeout(time: 5, unit: 'HOURS')
+    }
+
     tools {
         jdk 'JDK 1.8 (latest)'
     }

apm-webapp/pom.xml

@@ -37,6 +37,7 @@
         <apache-httpclient.version>4.5.3</apache-httpclient.version>
         <spring-cloud-dependencies.version>Edgware.SR1</spring-cloud-dependencies.version>
         <frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>
+        <logback-classic.version>1.2.3</logback-classic.version>
 
         <ui.path>${project.parent.basedir}/skywalking-ui</ui.path>
     </properties>
@@ -94,6 +95,13 @@
             <version>${spring.boot.version}</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback-classic.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

apm-webapp/src/test/java/org/apache/skywalking/apm/webapp/ApplicationContextTest.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.apm.webapp;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+/**
+ * @author kezhenxu94
+ */
+@SpringBootTest
+@RunWith(SpringRunner.class)
+public class ApplicationContextTest {
+
+    @Test
+    public void contextShouldLoad() {
+    }
+
+}

apm-webapp/src/test/java/org/apache/skywalking/apm/webapp/NotFoundHandlerTest.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.apm.webapp;
+
+import org.apache.skywalking.apm.webapp.proxy.NotFoundHandler;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author kezhenxu94
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(value = {NotFoundHandler.class, ClassPathResource.class})
+public class NotFoundHandlerTest {
+    @Mock
+    private NotFoundHandler notFoundHandler;
+
+    @Before
+    public void setUp() {
+        MockitoAnnotations.initMocks(this);
+    }
+
+    @Test
+    public void shouldInternalErrorWhenIndexPageIsMissing() throws Exception {
+        ClassPathResource mockIndexResource = mock(ClassPathResource.class);
+        when(mockIndexResource.getInputStream()).thenThrow(new IOException());
+
+        PowerMockito.whenNew(ClassPathResource.class)
+            .withArguments("/public/index.html")
+            .thenReturn(mockIndexResource);
+
+        when(notFoundHandler.renderDefaultPage()).thenCallRealMethod();
+        ResponseEntity<String> response = notFoundHandler.renderDefaultPage();
+        assertThat(response.getStatusCode()).isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR);
+    }
+}

apm-webapp/src/test/java/org/apache/skywalking/apm/webapp/WebAppTest.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.apm.webapp;
+
+import org.apache.skywalking.apm.webapp.proxy.NotFoundHandler;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.only;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+/**
+ * @author kezhenxu94
+ */
+@WebMvcTest
+@RunWith(SpringRunner.class)
+public class WebAppTest {
+    @Autowired
+    private MockMvc mvc;
+    @MockBean
+    private NotFoundHandler notFoundHandler;
+
+    @Test
+    public void shouldGetStaticResources() throws Exception {
+        when(notFoundHandler.renderDefaultPage()).thenCallRealMethod();
+
+        mvc.perform(get("/index.html"))
+            .andDo(print())
+            .andExpect(status().isOk())
+            .andExpect(content().string(containsString("<title>SkyWalking</title>")));
+
+        verify(notFoundHandler, never()).renderDefaultPage();
+    }
+
+    @Test
+    public void shouldRedirectToIndexWhenResourcesIsAbsent() throws Exception {
+        when(notFoundHandler.renderDefaultPage()).thenCallRealMethod();
+
+        mvc.perform(get("/absent.html"))
+            .andDo(print())
+            .andExpect(status().isOk());
+
+        verify(notFoundHandler, only()).renderDefaultPage();
+    }
+}

docker/oap/docker-entrypoint.sh

@@ -188,6 +188,17 @@ configuration:
 EOT
 }
 
+generateConfigurationConsul() {
+    cat <<EOT >> ${var_application_file}
+configuration:
+  consul:
+    # Consul host and ports, separated by comma, e.g. 1.2.3.4:8500,2.3.4.5:8500
+    hostAndPorts: \${SW_CONFIGURATION_CONSUL_ADDRESS:127.0.0.1:8500}
+    # Sync period in seconds. Defaults to 60 seconds.
+    period: \${SW_CONFIGURATION_CONSUL_PERIOD:60}
+EOT
+}
+
 generateTelemetryNone() {
     cat <<EOT >> ${var_application_file}
 telemetry:
@@ -346,6 +357,7 @@ EOT
     apollo) generateConfigurationApollo;;
     nacos) generateConfigurationNacos;;
     zookeeper) generateConfigurationZookeeper;;
+    consul) generateConfigurationConsul;;
     esac
 
     cat <<EOT >> ${var_application_file}

docs/README.md

@@ -23,6 +23,63 @@ Also, if you have some related to us, welcome to submit a pull request to add.
 if you see some problem. 
 Or better yet, submit your own contributions through pull request to help make them better.
 
+___
+# Document Catalog
+If you have been familiar with SkyWalking, you could use this catalog to find the document chapter directly.
+
+* [Concepts and Designs](en/concepts-and-designs/README.md)
+  * What is SkyWalking?
+    * [Overview and Core concepts](en/concepts-and-designs/overview.md). Provides a high-level description and introduction, including the problems the project solves.
+    * [Project Goals](en/concepts-and-designs/project-goals.md). Provides the goals, which SkyWalking is trying to focus and provide features about them.
+  * Probe
+    * [Introduction](en/concepts-and-designs/probe-introduction.md). Lead readers to understand what the probe is, how many different probes existed and why need them.
+    * [Service auto instrument agent](en/concepts-and-designs/service-agent.md). Introduce what the auto instrument agents do and which languages does SkyWalking already support. 
+    * [Manual instrument SDK](en/concepts-and-designs/manual-sdk.md). Introduce the role of the manual instrument SDKs in SkyWalking ecosystem.
+    * [Service Mesh probe](en/concepts-and-designs/service-mesh-probe.md). Introduce why and how SkyWalking receive telemetry data from Service mesh and proxy probe.
+  * Backend
+    * [Overview](en/concepts-and-designs/backend-overview.md). Provides a high level introduction about the OAP backend.
+    * [Observability Analysis Language](en/concepts-and-designs/oal.md). Introduces the core languages, which is designed for aggregation behaviour definition.
+    * [Query in OAP](en/protocols/README.md#query-protocol). A set of query protocol provided, based on the Observability Analysis Language metrics definition.
+  * UI
+    * [Overview](en/concepts-and-designs/ui-overview.md).  A simple brief about SkyWalking UI.
+* [Setup](en/setup/README.md).
+  * Backend, UI and Java agent are Apache official release, you could find them at [Apache SkyWalking DOWNLOAD page](http://skywalking.apache.org/downloads/).
+  * Language agents in Service
+    * [Java agent](en/setup/service-agent/java-agent/README.md). Introduce how to install java agent to your service, without change any codes.
+      * [Supported middleware, framework and library](en/setup/service-agent/java-agent/Supported-list.md).
+      * [Agent Configuration Properties](en/setup/service-agent/java-agent/README.md#table-of-agent-configuration-properties).
+      * [Optional plugins](en/setup/service-agent/java-agent/README.md#optional-plugins).
+      * [Bootstrap/JVM class plugin](en/setup/service-agent/java-agent/README.md#bootstrap-class-plugins).
+      * [Advanced features](en/setup/service-agent/java-agent/README.md#advanced-features).
+      * [Plugin development guide](en/setup/service-agent/java-agent/README.md#plugin-development-guide).
+      * [Agent plugin tests and performance tests](en/setup/service-agent/java-agent/README.md#test).
+    * [Other language agents](en/setup/README.md#language-agents-in-service) includes .NetCore, PHP, NodeJS, Go, which are maintained by volunteers.
+  * Service Mesh
+    * [SkyWalking on Istio](en/setup/istio/README.md). Introduce how to use Istio Mixer bypass Adapter to work with SkyWalking.
+    * Use [ALS(access log service)](https://www.envoyproxy.io/docs/envoy/latest/api-v2/service/accesslog/v2/als.proto) to observe service mesh, without Mixer. Follow [document](en/setup/envoy/als_setting.md) to open it.
+  * [Backend and UI setup document](en/setup/backend/backend-ui-setup.md).
+    * [Backend setup document](en/setup/backend/backend-setup.md).
+      * [Overriding settings](en/setup/backend/backend-setting-override.md) in application.yml is supported。
+      * [IP and port setting](en/setup/backend/backend-ip-port.md). Introduce how IP and port set and be used.
+      * [Backend init mode startup](en/setup/backend/backend-init-mode.md). How to init the environment and exit graciously. Read this before you try to initial a new cluster.
+      * [Cluster management](en/setup/backend/backend-cluster.md). Guide you to set backend server in cluster mode.
+      * [Deploy in kubernetes](en/setup/backend/backend-k8s.md). Guide you to build and use SkyWalking image, and deploy in k8s.
+      * [Choose storage](en/setup/backend/backend-storage.md). As we know, in default quick start, backend is running with H2 DB. But clearly, it doesn't fit the product env. In here, you could find what other choices do you have. Choose the one you like, we are also welcome anyone to contribute new storage implementor,
+      * [Set receivers](en/setup/backend/backend-receivers.md). You could choose receivers by your requirements, most receivers are harmless, at least our default receivers are. You would set and active all receivers provided.
+      * Do [trace sampling](en/setup/backend/trace-sampling.md) at backend. This sample keep the metrics accurate, only don't save some of traces in storage based on rate.
+      * Follow [slow DB statement threshold](en/setup/backend/slow-db-statement.md) config document to understand that, how to detect the Slow database statements(including SQL statements) in your system.
+      * Official [OAL scripts](en/guides/backend-oal-scripts.md). As you known from our [OAL introduction](en/concepts-and-designs/oal.md), most of backend analysis capabilities based on the scripts. Here is the description of official scripts, which helps you to understand which metrics data are in process, also could be used in alarm.
+      * [Alarm](en/setup/backend/backend-alarm.md). Alarm provides a time-series based check mechanism. You could set alarm  rules targeting the analysis oal metrics objects.
+      * [Advanced deployment options](en/setup/backend/advanced-deployment.md). If you want to deploy backend in very large scale and support high payload, you may need this. 
+      * [Metrics exporter](en/setup/backend/metrics-exporter.md). Use metrics data exporter to forward metrics data to 3rd party system.
+      * [Time To Live (TTL)](en/setup/backend/ttl.md). Metrics and trace are time series data, they would be saved forever, you could  set the expired time for each dimension.
+      * [Dynamic Configuration](en/setup/backend/dynamic-config.md). Make configuration of OAP changed dynamic, from remote service or 3rd party configuration management system.
+      * [Uninstrumented Gateways](en/setup/backend/uninstrumented-gateways.md). Configure gateways/proxies that are not supported by SkyWalking agent plugins, to reflect the delegation in topology graph.
+    * [UI setup document](en/setup/backend/ui-setup.md).
+* [Protocols](en/protocols/README.md). Protocols show the communication ways between agents/probes and backend. Anyone, interested in uplink telemetry data, definitely should read this.
+* [FAQs](en/FAQ/README.md). Include a manifest, including already known setup problems, secondary developments experiments. When  you are facing a problem, check here first.
+
+
 ___
 ### Users from 5.x
 SkyWalking 5.x is still supported by the community. For the user plans to upgrade from 5.x to 6.x, you should know there are some definitions of concepts changed.

docs/en/concepts-and-designs/manual-sdk.md

@@ -1,8 +1,8 @@
 # Manual instrument SDK
-We haven't provided any manual instrument SDK yet. 
+We have manual instrument SDK contributed from the community.
+- [Go2Sky](https://github.com/SkyAPM/go2sky). Go SDK follows SkyWalking format.
 
 Welcome to consider contributing in following languages:
-- Go
 - Python
 - C++
 

docs/en/concepts-and-designs/ui-overview.md

@@ -3,10 +3,6 @@ SkyWalking native UI provides the default visualization solution.
 It provides observability related graphs
 about overview, service, service instance, endpoint, trace and alarm, 
 including topology, dependency graph, heatmap, etc.
-
-For most open source users, this UI solution should be good enough to use.
-And you have other open source options in visualization.
-See [UI setup](../setup/backend/ui-setup.md).
 
 Also, we have already known, many of our users have integrated SkyWalking
 into their products. 

docs/en/setup/backend/backend-alarm.md

@@ -1,5 +1,5 @@
 # Alarm
-Alarm core is driven a collection of rules, which are defined in `config/alarm-settings.yml`.
+Alarm core is driven by a collection of rules, which are defined in `config/alarm-settings.yml`.
 There are two parts in alarm rule definition.
 1. [Alarm rules](#rules). They define how metrics alarm should be triggered, what conditions should be considered.
 1. [Webhooks](#webhook). The list of web service endpoint, which should be called after the alarm is triggered.
@@ -90,3 +90,11 @@ Example as following
 	"startTime": 1560524171000
 }]
 ```
+
+## Update the settings dynamically
+Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
+which will override the settings in `alarm-settings.yml`.
+
+In order to determine that whether an alarm rule is triggered or not, SkyWalking needs to cache the metrics of a time window for
+each alarm rule, if any attribute (`metrics-name`, `op`, `threshold`, `period`, `count`, etc.) of a rule is changed,
+the sliding window will be destroyed and re-created, causing the alarm of this specific rule to restart again.

docs/en/setup/backend/dynamic-config.md

@@ -7,7 +7,8 @@ Right now, SkyWalking supports following dynamic configurations.
 | Config Key | Value Description | Value Format Example |
 |:----:|:----:|:----:|
 |receiver-trace.default.slowDBAccessThreshold| Thresholds of slow Database statement, override `receiver-trace/default/slowDBAccessThreshold` of `applciation.yml`. | default:200,mongodb:50|
-|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | not set |
+|receiver-trace.default.uninstrumentedGateways| The uninstrumented gateways, override `gateways.yml`. | same as [`gateways.yml`](uninstrumented-gateways.md#configuration-format) |
+|alarm.default.alarm-settings| The alarm settings, will override `alarm-settings.yml`. | same as [`alarm-settings.yml`](backend-alarm.md) |
 
 
 This feature depends on upstream service, so it is **OFF** as default.
@@ -94,8 +95,18 @@ configuration:
     clusterName: "default"
 ```
 
-## 3rd party Configuration Center
-We are welcome contributions to implement this module provider to support popular configuration center, 
-such as Consul. Submit issue to discuss.
+## Dynamic Configuration Consul Implementation
+
+[Consul](https://github.com/rickfast/consul-client) is also supported as DCC(Dynamic Configuration Center), to use it, please configure as follows:
+
+```yaml
+configuration:
+  consul:
+    # Consul host and ports, separated by comma, e.g. 1.2.3.4:8500,2.3.4.5:8500
+    hostAndPorts: 127.0.0.1:8500
+    # Sync period in seconds. Defaults to 60 seconds.
+    period: 60
+```
+
 
 

docs/en/setup/service-agent/java-agent/README.md

@@ -32,7 +32,7 @@ The agent release dist is included in Apache [official release](http://skywalkin
 
 - Start your application.
 
-## Supported middlewares, frameworks and libraries
+## Supported middleware, framework and library
 SkyWalking agent has supported various middlewares, frameworks and libraries.
 Read [supported list](Supported-list.md) to get them and supported version.
 If the plugin is in **Optional²** catalog, go to [optional plugins](#optional-plugins) section to learn how to active it.