Merge branch 'master' into ci-pipeline

apache · Nov 21, 2019 · 8018bec · 8018bec
2 parents 3dd0284 + 9b408e8
commit 8018bec
Show file tree

Hide file tree

Showing 81 changed files with 2,967 additions and 426 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -72,7 +72,6 @@ pipeline {
                                 }
                             }
                             steps {
-                                sh 'tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist'
                                 sh 'tools/dependencies/check-LICENSE.sh'
                             }
                         }

diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E
@@ -113,6 +113,12 @@ pipeline {
                                 sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-agent-reboot'
                             }
                         }
+
+                        stage('Cluster Tests (ES7/ZK/JDK8)') {
+                            steps {
+                                sh 'E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.0.0 bash -x test/e2e/run.sh e2e-cluster/test-runner'
+                            }
+                        }
                     }
                 }
 
@@ -129,6 +135,12 @@ pipeline {
                                 sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es'
                             }
                         }
+
+                        stage('TTL ES7 Tests(JDK8)') {
+                            steps {
+                                sh 'E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.0.0 bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es'
+                            }
+                        }
                     }
                 }
             }

diff --git a/apm-dist/pom.xml b/apm-dist/pom.xml
@@ -71,7 +71,6 @@
     </profiles>
 
     <build>
-        <finalName>apache-skywalking-apm-bin</finalName>
         <plugins>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
@@ -82,22 +81,39 @@
                         <goals>
                             <goal>single</goal>
                         </goals>
+                        <configuration>
+                            <finalName>apache-skywalking-apm-bin</finalName>
+                            <descriptors>
+                                <descriptor>${project.basedir}/src/main/assembly/binary.xml</descriptor>
+                            </descriptors>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>dist-es7</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>apache-skywalking-apm-bin-es7</finalName>
+                            <descriptors>
+                                <descriptor>${project.basedir}/src/main/assembly/binary-es7.xml</descriptor>
+                            </descriptors>
+                        </configuration>
                     </execution>
                 </executions>
                 <configuration>
                     <attach>true</attach>
                     <tarLongFileMode>posix</tarLongFileMode>
                     <runOnlyAtExecutionRoot>false</runOnlyAtExecutionRoot>
-                    <descriptors>
-                        <descriptor>${project.basedir}/src/main/assembly/binary.xml</descriptor>
-                    </descriptors>
                     <appendAssemblyId>false</appendAssemblyId>
                 </configuration>
             </plugin>
             <plugin>
                 <artifactId>maven-antrun-plugin</artifactId>
                 <executions>
                     <execution>
+                        <id>dist</id>
                         <phase>package</phase>
                         <goals>
                             <goal>run</goal>
@@ -109,6 +125,19 @@
                             </tasks>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>dist-es7</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy file="${project.build.directory}/apache-skywalking-apm-bin-es7.tar.gz" tofile="${project.basedir}/../dist/apache-skywalking-apm-bin-es7.tar.gz" overwrite="true" />
+                                <copy file="${project.build.directory}/apache-skywalking-apm-bin-es7.zip" tofile="${project.basedir}/../dist/apache-skywalking-apm-bin-es7.zip" overwrite="true" />
+                            </tasks>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
         </plugins>

diff --git a/apm-dist/release-docs/LICENSE b/apm-dist/release-docs/LICENSE
@@ -238,6 +238,7 @@ The text of each license is the standard Apache 2.0 license.
     Google: proto-google-common-protos 0.1.9: https://github.com/googleapis/googleapis , Apache 2.0
     Google: jsr305 3.0.0: http://central.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.pom , Apache 2.0
     Elasticsearch BV (Elasticsearch) 6.3.2: https://www.elastic.co/products/elasticsearch , Apache 2.0
+    Elasticsearch BV (Elasticsearch) 7.0.0: https://www.elastic.co/products/elasticsearch , Apache 2.0
     lang-mustache-client 5.5.0: https://github.com/elastic/elasticsearch/tree/master/modules/lang-mustache , Apache 2.0
     parent-join-client 5.5.0: https://github.com/elastic/elasticsearch/tree/master/modules/parent-join , Apache 2.0
     reindex-client 5.5.0: https://github.com/elastic/elasticsearch/tree/master/modules/reindex , Apache 2.0

diff --git a/...c/main/assembly/alarm-settings-sample.yml → ...c/main/assembly/alarm-settings-sample.yml b/...c/main/assembly/alarm-settings-sample.yml → ...c/main/assembly/alarm-settings-sample.yml
diff --git a/...rter/src/main/assembly/alarm-settings.yml → ...dist/src/main/assembly/alarm-settings.yml b/...rter/src/main/assembly/alarm-settings.yml → ...dist/src/main/assembly/alarm-settings.yml
diff --git a/...starter/src/main/assembly/application.yml → apm-dist/src/main/assembly/application.yml b/...starter/src/main/assembly/application.yml → apm-dist/src/main/assembly/application.yml
@@ -95,6 +95,28 @@ storage:
 #    resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000}
 #    metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000}
 #    segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200}
+#  elasticsearch7:
+#    nameSpace: ${SW_NAMESPACE:""}
+#    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
+#    protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"}
+#    trustStorePath: ${SW_SW_STORAGE_ES_SSL_JKS_PATH:"../es_keystore.jks"}
+#    trustStorePass: ${SW_SW_STORAGE_ES_SSL_JKS_PASS:""}
+#    user: ${SW_ES_USER:""}
+#    password: ${SW_ES_PASSWORD:""}
+#    indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
+#    indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
+#    # Those data TTL settings will override the same settings in core module.
+#    recordDataTTL: ${SW_STORAGE_ES_RECORD_DATA_TTL:7} # Unit is day
+#    otherMetricsDataTTL: ${SW_STORAGE_ES_OTHER_METRIC_DATA_TTL:45} # Unit is day
+#    monthMetricsDataTTL: ${SW_STORAGE_ES_MONTH_METRIC_DATA_TTL:18} # Unit is month
+#    # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html
+#    bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests
+#    flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
+#    concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
+#    resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000}
+#    metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000}
+#    segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200}
+#    indexMaxResultWindow: ${SW_STORAGE_ES_INDEX_MAX_RESULT_WINDOW:5000}
   h2:
     driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}
     url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db}

diff --git a/apm-dist/src/main/assembly/binary-es7.xml b/apm-dist/src/main/assembly/binary-es7.xml
@@ -0,0 +1,105 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <id>dist</id>
+    <formats>
+        <format>zip</format>
+        <format>tar.gz</format>
+    </formats>
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/bin</directory>
+            <outputDirectory>/bin</outputDirectory>
+            <includes>
+                <include>*.sh</include>
+                <include>*.bat</include>
+            </includes>
+            <fileMode>0755</fileMode>
+        </fileSet>
+        <fileSet>
+            <directory>src/main/assembly</directory>
+            <outputDirectory>/config</outputDirectory>
+            <includes>
+                <include>log4j2.xml</include>
+                <include>application.yml</include>
+                <include>alarm-settings.yml</include>
+                <include>alarm-settings-sample.yml</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../oap-server/server-bootstrap/src/main/resources</directory>
+            <includes>
+                <include>component-libraries.yml</include>
+                <include>official_analysis.oal</include>
+                <include>gateways.yml</include>
+            </includes>
+            <outputDirectory>/config</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../oap-server/server-starter-es7/target/skywalking-oap-assembly/skywalking-oap/config</directory>
+            <outputDirectory>/config</outputDirectory>
+            <includes>
+                <include>*.yml</include>
+                <include>*.xml</include>
+                <include>*.properties</include>
+                <include>*.oal</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../oap-server/server-starter-es7/target/skywalking-oap-assembly/skywalking-oap/libs</directory>
+            <outputDirectory>/oap-libs</outputDirectory>
+        </fileSet>
+
+        <!-- Agent repackage into the dist -->
+        <fileSet>
+            <directory>${project.basedir}/../skywalking-agent</directory>
+            <outputDirectory>/agent</outputDirectory>
+        </fileSet>
+
+        <!-- Release docs and licenses -->
+        <fileSet>
+            <directory>${project.basedir}/../</directory>
+            <outputDirectory>/</outputDirectory>
+            <includes>
+                <include>DISCLAIMER</include>
+                <include>CHANGES</include>
+            </includes>
+        </fileSet>
+
+        <fileSet>
+            <directory>${project.basedir}/release-docs</directory>
+            <outputDirectory>/</outputDirectory>
+        </fileSet>
+    </fileSets>
+    <files>
+        <file>
+            <source>${project.basedir}/../apm-webapp/target/skywalking-webapp.jar</source>
+            <outputDirectory>/webapp</outputDirectory>
+            <fileMode>0644</fileMode>
+        </file>
+        <file>
+            <source>${project.basedir}/../apm-webapp/src/main/assembly/webapp.yml</source>
+            <outputDirectory>/webapp</outputDirectory>
+            <fileMode>0644</fileMode>
+        </file>
+    </files>
+</assembly>
diff --git a/apm-dist/src/main/assembly/binary.xml b/apm-dist/src/main/assembly/binary.xml
@@ -35,6 +35,25 @@
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>
+        <fileSet>
+            <directory>src/main/assembly</directory>
+            <outputDirectory>/config</outputDirectory>
+            <includes>
+                <include>log4j2.xml</include>
+                <include>application.yml</include>
+                <include>alarm-settings.yml</include>
+                <include>alarm-settings-sample.yml</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/../oap-server/server-bootstrap/src/main/resources</directory>
+            <includes>
+                <include>component-libraries.yml</include>
+                <include>official_analysis.oal</include>
+                <include>gateways.yml</include>
+            </includes>
+            <outputDirectory>/config</outputDirectory>
+        </fileSet>
         <fileSet>
             <directory>${project.basedir}/../oap-server/server-starter/target/skywalking-oap-assembly/skywalking-oap/config</directory>
             <outputDirectory>/config</outputDirectory>

diff --git a/...rver-starter/src/main/assembly/log4j2.xml → apm-dist/src/main/assembly/log4j2.xml b/...rver-starter/src/main/assembly/log4j2.xml → apm-dist/src/main/assembly/log4j2.xml
diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md
@@ -4,7 +4,7 @@ use is by changing the `application.yml`
 
 Native supported storage
 - H2
-- ElasticSearch 6
+- ElasticSearch 6, 7
 - MySQL
 - TiDB
 
@@ -26,10 +26,16 @@ storage:
     user: sa
 ```
 
-## ElasticSearch 6
-Active ElasticSearch 6 as storage, set storage provider to **elasticsearch**.
+## ElasticSearch
+- In order to activate ElasticSearch 6 as storage, set storage provider to **elasticsearch**
+- In order to activate ElasticSearch 7 as storage, set storage provider to **elasticsearch7**
 
-**Required ElasticSearch 6.3.2 or higher, excepted 7.0.0 or higher. HTTP RestHighLevelClient is used to connect server.**
+**Required ElasticSearch 6.3.2 or higher. HTTP RestHighLevelClient is used to connect server.**
+
+- For ElasticSearch 6.3.2 ~ 7.0.0 (excluded), please download the `apache-skywalking-bin.tar.gz` or `apache-skywalking-bin.zip`,
+- For ElasticSearch 7.0.0 ~ 8.0.0 (excluded), please download the `apache-skywalking-bin-es7.tar.gz` or `apache-skywalking-bin-es7.zip`.
+
+ElasticSearch 6 and ElasticSearch 7 share most of the configurations, as follows:
 
 Setting fragment example
 
@@ -54,6 +60,22 @@ storage:
     bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20} # flush the bulk every 20mb
     flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests
     concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
+    resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000}
+    metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000}
+    segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200}
+```
+
+and there're also some configurations that are ES7 specific, as follows:
+
+```yaml
+storage:
+  elasticsearch7:
+    # ... the configurations shared with ES6 that are listed above ...
+
+    # Index max result window, for segment deep pagination, usually we don't recommend to scroll too many pages,
+    # instead, give more query criteria (e.g. service id or time range), to narrow the query results.
+    # see https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html for more information
+    indexMaxResultWindow: ${SW_STORAGE_ES_INDEX_MAX_RESULT_WINDOW:5000}
 ```
 
 ### ElasticSearch 6 With Https SSL Encrypting communications.
@@ -87,9 +109,19 @@ storage:
 ### Data TTL
 TTL in ElasticSearch overrides the settings of core, read [ElasticSearch section in TTL document](ttl.md#elasticsearch-6-storage-ttl)
 
-### ElasticSearch server settings
-Read the [ElasticSearch storage FAQ](../../FAQ/ES-Server-FAQ.md) if you are new to ElasticSearch. 
-And recommend read more about these configuration from ElasticSearch official document. 
+### Recommended ElasticSearch server-side configurations
+You could add following config to `elasticsearch.yml`, set the value based on your env.
+
+```yml
+# In tracing scenario, consider to set more than this at least.
+thread_pool.index.queue_size: 1000 # Only suitable for ElasticSearch 6
+thread_pool.write.queue_size: 1000 # Suitable for ElasticSearch 6 and 7
+
+# When you face query error at trace page, remember to check this.
+index.max_result_window: 1000000 # Only suitable for ElasticSearch 6. For ES 7, set `indexMaxResultWindow` under `storage`-`elasticsearch7` section in application.yml
+```
+
+We strongly advice you to read more about these configurations from ElasticSearch official document. 
 This effects the performance of ElasticSearch very much.
 
 
@@ -194,7 +226,7 @@ These settings can refer to the configuration of *MySQL* above.
 
 ## ElasticSearch 5
 ElasticSearch 5 is incompatible with ElasticSearch 6 Java client jar, so it could not be included in native distribution.
-[OpenSkywalking/SkyWalking-With-Es5x-Storage](https://github.com/OpenSkywalking/SkyWalking-With-Es5x-Storage) repo includes the distribution version. 
+[OpenSkyWalking/SkyWalking-With-Es5x-Storage](https://github.com/OpenSkywalking/SkyWalking-With-Es5x-Storage) repo includes the distribution version. 
 
 ## More storage solution extension
 Follow [Storage extension development guide](../../guides/storage-extention.md) 

diff --git a/docs/en/setup/service-agent/java-agent/README.md b/docs/en/setup/service-agent/java-agent/README.md
@@ -53,12 +53,19 @@ Change the first line of `tomcat/bin/catalina.bat`.
 ```shell
 set "CATALINA_OPTS=-javaagent:/path/to/skywalking-agent/skywalking-agent.jar"
 ```
+
 - JAR file  
 Add `-javaagent` argument to command line in which you start your app. eg:
  ```shell
  java -javaagent:/path/to/skywalking-agent/skywalking-agent.jar -jar yourApp.jar
  ```
-
+
+- Jetty  
+Modify `jetty.sh`, add `-javaagent` argument to command line in which you start your app. eg:
+```shell
+export JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:/path/to/skywalking-agent/skywalking-agent.jar"
+```
+
 ## Table of Agent Configuration Properties
 This is the properties list supported in `agent/config/agent.config`.
 

diff --git a/oap-server/pom.xml b/oap-server/pom.xml
@@ -34,6 +34,7 @@
         <module>server-storage-plugin</module>
         <module>server-library</module>
         <module>server-starter</module>
+        <module>server-starter-es7</module>
         <module>server-query-plugin</module>
         <module>server-alarm-plugin</module>
         <module>server-testing</module>
@@ -42,6 +43,7 @@
         <module>oal-grammar</module>
         <module>exporter</module>
         <module>server-configuration</module>
+        <module>server-bootstrap</module>
     </modules>
 
     <properties>