Skip to content

Commit

Permalink
Merge branch 'master' into auto-refresh
Browse files Browse the repository at this point in the history
  • Loading branch information
tomsun28 committed Jun 14, 2024
2 parents 54ccfd0 + 365de46 commit 2920a7c
Show file tree
Hide file tree
Showing 430 changed files with 25,500 additions and 6,960 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,13 @@ Detailed config refer to [Install HertzBeat via Package](https://hertzbeat.apach

Detailed steps refer to [CONTRIBUTING](CONTRIBUTING.md)

##### 4:Install All(hertzbeat+mysql+tsdb) via Docker-compose
##### 4:Install All(hertzbeat+postgresql+tsdb) via Docker-compose

Install the mysql/postgresql database, iotdb/tdengine/victoria-metrics database and hertzbeat at one time through [docker-compose deployment script](script/docker-compose).
Install the postgresql/mysql database, victoria-metrics/iotdb/tdengine database and hertzbeat at one time through [docker-compose deployment script](script/docker-compose).

Detailed steps refer to [Install via Docker-Compose](script/docker-compose/README.md)

##### 5. Install All(hertzbeat+collector+mysql+tsdb) via kubernetes helm charts
##### 5. Install All(hertzbeat+collector+postgresql+tsdb) via kubernetes helm charts

Install HertzBeat cluster in a Kubernetes cluster by Helm chart.

Expand Down
6 changes: 3 additions & 3 deletions README_CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,13 @@ docker run -d -e IDENTITY=custom-collector-name -e MANAGER_HOST=127.0.0.1 -e MAN

详细步骤参考 [参与贡献之本地代码启动](CONTRIBUTING.md)

##### 方式四:Docker-Compose 统一安装 hertzbeat+mysql+tsdb
##### 方式四:Docker-Compose 统一安装 hertzbeat+postgresql+tsdb

通过 [Docker-Compose 部署脚本](script/docker-compose) 一次性把 mysql/postgresql 数据库, iotdb/tdengine/victoria-metrics 时序数据库和 hertzbeat 安装部署。
通过 [Docker-Compose 部署脚本](script/docker-compose) 一次性把 postgresql/mysql 数据库, victoria-metrics/iotdb/tdengine 时序数据库和 hertzbeat 安装部署。

详细步骤参考 [通过Docker-Compose安装HertzBeat](script/docker-compose/README.md)

##### 方式五:Kubernetes Helm Charts 部署 hertzbeat+collector+mysql+tsdb
##### 方式五:Kubernetes Helm Charts 部署 hertzbeat+collector+postgresql+tsdb

通过 Helm Chart 一次性将 HertzBeat 集群组件部署到 Kubernetes 集群中。

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ private DnsResolveResult dig(DnsProtocol dns) throws IOException {
responseTimeStopWatch.start();

Name name = Name.fromString(dns.getAddress(), Name.root);
Message query = Message.newQuery(Record.newRecord(name, Type.ANY, DClass.ANY));
Message query = Message.newQuery(Record.newRecord(name, Type.ANY, DClass.value(dns.getQueryClass())));
Resolver res = new SimpleResolver(dns.getDnsServerIP());
res.setTimeout(Duration.of(Long.parseLong(dns.getTimeout()), ChronoUnit.MILLIS));
res.setTCP(Boolean.parseBoolean(dns.getTcp()));
Expand Down Expand Up @@ -195,7 +195,7 @@ private Map<String, String> getHeaderInfo(Message message, Long responseTime) {

private List<String> getSectionInfo(Message message, int section) {
List<RRset> currentSetList = message.getSectionRRsets(section);

if (CollectionUtils.isEmpty(currentSetList)) {
return Lists.newArrayList();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hertzbeat.collector.collect.imap;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.net.imap.IMAPClient;
import org.apache.commons.net.imap.IMAPSClient;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.ImapProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.springframework.util.Assert;

/**
* imap collect
*/
@Slf4j
public class ImapCollectImpl extends AbstractCollect {

private static final String UTF_7_X = "X-MODIFIED-UTF-7";
private static final String STATUS = "STATUS";
private static final String STATUS_COMMAND = "(MESSAGES RECENT UNSEEN)";
private static final String MESSAGES = "MESSAGES";
private static final String RECENT = "RECENT";
private static final String UNSEEN = "UNSEEN";
private static final String RESPONSETIME = "responseTime";
private static final String totalMessageCount = "TotalMessageCount";
private static final String recentMessageCount = "RecentMessageCount";
private static final String unseenMessageCount = "UnseenMessageCount";

@Override
public void preCheck(Metrics metrics) throws IllegalArgumentException {
ImapProtocol imapProtocol = metrics.getImap();
Assert.notNull(metrics, "IMAP collect must has Imap params");
Assert.notNull(metrics.getImap(), "IMAP collect must has Imap params");
Assert.hasText(imapProtocol.getHost(), "IMAP host is required");
Assert.hasText(imapProtocol.getPort(), "IMAP port is required");
Assert.hasText(imapProtocol.getEmail(), "IMAP email is required");
Assert.hasText(imapProtocol.getAuthorize(), "IMAP authorize code is required");
Assert.hasText(imapProtocol.getFolderName(), "IMAP folder name is required");
}

@Override
public void collect(CollectRep.MetricsData.Builder builder, long monitorId, String app, Metrics metrics) {
long startTime = System.currentTimeMillis();
ImapProtocol imapProtocol = metrics.getImap();
IMAPClient imapClient = null;
boolean ssl = Boolean.parseBoolean(imapProtocol.getSsl());

try {
imapClient = createImapClient(imapProtocol, ssl);
// if Connected, then collect metrics
if (imapClient.isConnected()) {
long responseTime = System.currentTimeMillis() - startTime;
String folderName = imapProtocol.getFolderName();
collectImapMetrics(builder, imapClient, metrics.getAliasFields(), folderName, responseTime);
} else {
builder.setCode(CollectRep.Code.UN_CONNECTABLE);
builder.setMsg("Peer connect failed,Timeout " + imapProtocol.getTimeout() + "ms");
}
} catch (Exception e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
} finally {
if (imapClient != null) {
try {
imapClient.logout();
imapClient.disconnect();
} catch (IOException e) {
String errorMsg = CommonUtil.getMessageFromThrowable(e);
log.error(errorMsg);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg(errorMsg);
}
}
}
}

@Override
public String supportProtocol() {
return DispatchConstants.PROTOCOL_IMAP;
}

private IMAPClient createImapClient(ImapProtocol imapProtocol, boolean ssl) throws Exception {
IMAPClient imapClient = null;
// determine whether to use SSL-encrypted connections
imapClient = new IMAPSClient(true);
if (!ssl) {
imapClient = new IMAPClient();
}
// set timeout
int timeout = Integer.parseInt(imapProtocol.getTimeout());
if (timeout > 0) {
imapClient.setConnectTimeout(timeout);
}
//set Charset
imapClient.setCharset(StandardCharsets.US_ASCII);
// connect to the IMAP server
String host = imapProtocol.getHost();
int port = Integer.parseInt(imapProtocol.getPort());
imapClient.connect(host, port);
// validate credentials
String email = imapProtocol.getEmail();
String authorize = imapProtocol.getAuthorize();
boolean isAuthenticated = imapClient.login(email, authorize);
if (!isAuthenticated) {
throw new Exception("IMAP client authentication failed");
}
return imapClient;

}

private void collectImapMetrics(CollectRep.MetricsData.Builder builder, IMAPClient imapClient, List<String> aliasFields,
String folderName, long responseTime) throws Exception {
Map<String, String> resultsMap = new HashMap<>();
resultsMap.put(RESPONSETIME, String.valueOf(responseTime));
imapClient.sendCommand(STATUS + " \"" + CollectUtil.stringEncodeUtf7String(folderName, UTF_7_X) + "\" " + STATUS_COMMAND);
String[] response = imapClient.getReplyString().split("\\s+|\\(|\\)");
for (int i = 0; i < response.length; i++) {
switch (response[i]) {
case MESSAGES:
resultsMap.put(folderName + totalMessageCount, response[i + 1]);
break;
case RECENT:
resultsMap.put(folderName + recentMessageCount, response[i + 1]);
break;
case UNSEEN:
resultsMap.put(folderName + unseenMessageCount, response[i + 1]);
break;
default:
break;
}
}

CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultsMap.get(field);
valueRowBuilder.addColumns(Objects.requireNonNullElse(fieldValue, CommonConstants.NULL_VALUE));
}
builder.addValues(valueRowBuilder.build());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hertzbeat.collector.collect.common.cache.ConnectionCommonCache;
import org.apache.hertzbeat.collector.collect.common.cache.MongodbConnect;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CollectorConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.MongodbProtocol;
Expand Down Expand Up @@ -188,11 +189,20 @@ private MongoClient getClient(Metrics metrics, CacheIdentifier identifier) {
if (mongoClient != null) {
return mongoClient;
}
// If the multiplexing fails, create a new connection to connect to mongodb
// Passwords may contain special characters and need to be encoded using JS-like encodeURIComponent, which uses java URLEncoder
String url = String.format("mongodb://%s:%s@%s:%s/%s?authSource=%s", mongodbProtocol.getUsername(),
URLEncoder.encode(mongodbProtocol.getPassword(), StandardCharsets.UTF_8), mongodbProtocol.getHost(), mongodbProtocol.getPort(),
mongodbProtocol.getDatabase(), mongodbProtocol.getAuthenticationDatabase());

String url = null;
if (CollectorConstants.MONGO_DB_ATLAS_MODEL.equals(mongodbProtocol.getModel())){
url = String.format("mongodb+srv://%s:%s@%s/%s?authSource=%s", mongodbProtocol.getUsername(),
URLEncoder.encode(mongodbProtocol.getPassword(), StandardCharsets.UTF_8), mongodbProtocol.getHost(),
mongodbProtocol.getDatabase(), mongodbProtocol.getAuthenticationDatabase());
} else {
// If the multiplexing fails, create a new connection to connect to mongodb
// Passwords may contain special characters and need to be encoded using JS-like encodeURIComponent, which uses java URLEncoder
url = String.format("mongodb://%s:%s@%s:%s/%s?authSource=%s", mongodbProtocol.getUsername(),
URLEncoder.encode(mongodbProtocol.getPassword(), StandardCharsets.UTF_8), mongodbProtocol.getHost(), mongodbProtocol.getPort(),
mongodbProtocol.getDatabase(), mongodbProtocol.getAuthenticationDatabase());
}

// Use the Mongo Client Settings builder to configure timeouts and other configurations
MongoClientSettings settings = MongoClientSettings.builder()
.applyConnectionString(new ConnectionString(url))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ public interface DispatchConstants {
* protocol redfish
*/
String PROTOCOL_REDFISH = "redfish";
/**
* protocol imap
*/
String PROTOCOL_IMAP = "imap";

// Protocol type related - end

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@

package org.apache.hertzbeat.collector.util;

import com.beetstra.jutf7.CharsetProvider;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -455,4 +457,24 @@ public static byte[] fromHexString(String hexString) {
}
return bytes;
}

/**
* convert original string to UTF-7 String
* @param original original text
* @param charset encode charset
* @return String
*/
public static String stringEncodeUtf7String(String original, String charset) {
return new String(original.getBytes(new CharsetProvider().charsetForName(charset)), StandardCharsets.US_ASCII);
}

/**
* convert UTF-7 string to original String
* @param encoded encoded String
* @param charset encode charset
* @return String
*/
public static String utf7StringDecodeString(String encoded, String charset) {
return new String(encoded.getBytes(StandardCharsets.US_ASCII), new CharsetProvider().charsetForName(charset));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ org.apache.hertzbeat.collector.collect.pop3.Pop3CollectImpl
org.apache.hertzbeat.collector.collect.httpsd.HttpsdImpl
org.apache.hertzbeat.collector.collect.redfish.RedfishCollectImpl
org.apache.hertzbeat.collector.collect.nebulagraph.NgqlCollectImpl
org.apache.hertzbeat.collector.collect.imap.ImapCollectImpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
package org.apache.hertzbeat.collector.collect.dns;


import static org.junit.jupiter.api.Assertions.assertNotNull;

import java.util.Collections;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.DnsProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
Expand All @@ -40,6 +43,8 @@ public void setup() {
.dnsServerIP("8.8.8.8")
.queryClass("IN")
.address("www.google.com")
.timeout("3000")
.port("53")
.build();
}

Expand All @@ -49,8 +54,10 @@ public void testCollect() {
long monitorId = 666;
String app = "testDNS";
Metrics metrics = new Metrics();
metrics.setName("question");
metrics.setDns(dnsProtocol);

metrics.setAliasFields(Collections.singletonList("section"));
dnsCollect.collect(builder, monitorId, app, metrics);
assertNotNull(builder.getValues(0).getColumns(0));
}
}
Loading

0 comments on commit 2920a7c

Please sign in to comment.