Skip to content
Permalink
Browse files
Add 'exclude' filtering policy for instance-level params (#10015)
  • Loading branch information
chickenlj committed May 10, 2022
1 parent ec98ff3 commit fe2d7dcd040b116badea176fc17e85a1f8fd00db
Showing 15 changed files with 466 additions and 74 deletions.
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dubbo.metadata;

import org.apache.dubbo.common.extension.Activate;

import static org.apache.dubbo.common.constants.CommonConstants.MONITOR_KEY;
import static org.apache.dubbo.common.constants.CommonConstants.PID_KEY;
import static org.apache.dubbo.common.constants.CommonConstants.TIMESTAMP_KEY;
import static org.apache.dubbo.common.constants.FilterConstants.VALIDATION_KEY;
import static org.apache.dubbo.common.constants.QosConstants.ACCEPT_FOREIGN_IP;
import static org.apache.dubbo.common.constants.QosConstants.QOS_ENABLE;
import static org.apache.dubbo.common.constants.QosConstants.QOS_HOST;
import static org.apache.dubbo.common.constants.QosConstants.QOS_PORT;
import static org.apache.dubbo.remoting.Constants.BIND_IP_KEY;
import static org.apache.dubbo.remoting.Constants.BIND_PORT_KEY;
import static org.apache.dubbo.remoting.Constants.HEARTBEAT_TIMEOUT_KEY;
import static org.apache.dubbo.rpc.Constants.INTERFACES;

@Activate
public class DefaultMetadataParamsFilter implements MetadataParamsFilter {
private final String[] excludedServiceParams;
private final String[] includedInstanceParams;

public DefaultMetadataParamsFilter() {
this.includedInstanceParams = new String[]{HEARTBEAT_TIMEOUT_KEY};
this.excludedServiceParams = new String[]{MONITOR_KEY, BIND_IP_KEY, BIND_PORT_KEY, QOS_ENABLE,
QOS_HOST, QOS_PORT, ACCEPT_FOREIGN_IP, VALIDATION_KEY, INTERFACES, PID_KEY, TIMESTAMP_KEY};
}

@Override
public String[] instanceParamsIncluded() {
return includedInstanceParams;
}

@Override
public String[] serviceParamsExcluded() {
return excludedServiceParams;
}
}
@@ -27,6 +27,7 @@
import org.apache.dubbo.common.utils.StringUtils;

import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -46,18 +47,7 @@
import static org.apache.dubbo.common.constants.CommonConstants.DOT_SEPARATOR;
import static org.apache.dubbo.common.constants.CommonConstants.GROUP_CHAR_SEPARATOR;
import static org.apache.dubbo.common.constants.CommonConstants.METHODS_KEY;
import static org.apache.dubbo.common.constants.CommonConstants.MONITOR_KEY;
import static org.apache.dubbo.common.constants.CommonConstants.PID_KEY;
import static org.apache.dubbo.common.constants.CommonConstants.TIMESTAMP_KEY;
import static org.apache.dubbo.common.constants.FilterConstants.VALIDATION_KEY;
import static org.apache.dubbo.common.constants.QosConstants.ACCEPT_FOREIGN_IP;
import static org.apache.dubbo.common.constants.QosConstants.QOS_ENABLE;
import static org.apache.dubbo.common.constants.QosConstants.QOS_HOST;
import static org.apache.dubbo.common.constants.QosConstants.QOS_PORT;
import static org.apache.dubbo.metadata.RevisionResolver.EMPTY_REVISION;
import static org.apache.dubbo.remoting.Constants.BIND_IP_KEY;
import static org.apache.dubbo.remoting.Constants.BIND_PORT_KEY;
import static org.apache.dubbo.rpc.Constants.INTERFACES;

public class MetadataInfo implements Serializable {
public static final MetadataInfo EMPTY = new MetadataInfo();
@@ -367,24 +357,46 @@ private void extractInstanceParams(URL url, List<MetadataParamsFilter> filters)
return;
}

filters.forEach(filter -> {
String[] included = filter.instanceParamsIncluded();
if (ArrayUtils.isEmpty(included)) {
/*
* Does not put any parameter in instance if not specified.
* It will bring no functional suppression as long as all params will appear in service metadata.
*/
} else {
for (String p : included) {
String value = url.getParameter(p);
if (value != null) {
String oldValue = instanceParams.put(p, value);
if (oldValue != null && !oldValue.equals(value)) {
throw new IllegalStateException(String.format("Inconsistent instance metadata found in different services: %s, %s", oldValue, value));
}
}
String[] included, excluded;
if (filters.size() == 1) {
MetadataParamsFilter filter = filters.get(0);
included = filter.instanceParamsIncluded();
excluded = filter.instanceParamsExcluded();
} else {
Set<String> includedList = new HashSet<>();
Set<String> excludedList = new HashSet<>();
filters.forEach(filter -> {
if (ArrayUtils.isNotEmpty(filter.instanceParamsIncluded())) {
includedList.addAll(Arrays.asList(filter.instanceParamsIncluded()));
}
if (ArrayUtils.isNotEmpty(filter.instanceParamsExcluded())) {
excludedList.addAll(Arrays.asList(filter.instanceParamsExcluded()));
}
});
included = includedList.toArray(new String[0]);
excluded = excludedList.toArray(new String[0]);
}

Map<String, String> tmpInstanceParams = new HashMap<>();
if (ArrayUtils.isNotEmpty(included)) {
for (String p : included) {
String value = url.getParameter(p);
if (value != null) {
tmpInstanceParams.put(p, value);
}
}
} else if (ArrayUtils.isNotEmpty(excluded)) {
tmpInstanceParams.putAll(url.getParameters());
for (String p : excluded) {
tmpInstanceParams.remove(p);
}
}

tmpInstanceParams.forEach((key, value) -> {
String oldValue = instanceParams.put(key, value);
if (oldValue != null && !oldValue.equals(value)) {
throw new IllegalStateException(String.format("Inconsistent instance metadata found in different services: %s, %s", oldValue, value));
}
});
}

@@ -442,42 +454,12 @@ public static class ServiceInfo implements Serializable {

private transient URL url;

private final static String[] KEYS_TO_REMOVE = {MONITOR_KEY, BIND_IP_KEY, BIND_PORT_KEY, QOS_ENABLE,
QOS_HOST, QOS_PORT, ACCEPT_FOREIGN_IP, VALIDATION_KEY, INTERFACES, PID_KEY, TIMESTAMP_KEY};

public ServiceInfo() {}

public ServiceInfo(URL url, List<MetadataParamsFilter> filters) {
this(url.getServiceInterface(), url.getGroup(), url.getVersion(), url.getProtocol(), url.getPath(), null);
this.url = url;
Map<String, String> params = new HashMap<>();
if (filters.size() == 0) {
params.putAll(url.getParameters());
for (String key : KEYS_TO_REMOVE) {
params.remove(key);
}
}
for (MetadataParamsFilter filter : filters) {
String[] paramsIncluded = filter.serviceParamsIncluded();
if (ArrayUtils.isNotEmpty(paramsIncluded)) {
for (String p : paramsIncluded) {
String value = url.getParameter(p);
if (StringUtils.isNotEmpty(value) && params.get(p) == null) {
params.put(p, value);
}
String[] methods = url.getParameter(METHODS_KEY, (String[]) null);
if (methods != null) {
for (String method : methods) {
String mValue = url.getMethodParameterStrict(method, p);
if (StringUtils.isNotEmpty(mValue)) {
params.put(method + DOT_SEPARATOR + p, mValue);
}
}
}
}
}
}
this.params = params;
Map<String, String> params = extractServiceParams(url, filters);
// initialize method params caches.
this.methodParams = URLParam.initMethodParameters(params);
this.consumerMethodParams = URLParam.initMethodParameters(consumerParams);
@@ -495,6 +477,75 @@ public ServiceInfo(String name, String group, String version, String protocol, S
this.matchKey = buildMatchKey();
}

private Map<String, String> extractServiceParams(URL url, List<MetadataParamsFilter> filters) {
Map<String, String> params = new HashMap<>();

if (CollectionUtils.isEmpty(filters)) {
params.putAll(url.getParameters());
this.params = params;
return params;
}

String[] included, excluded;
if (filters.size() == 1) {
included = filters.get(0).serviceParamsIncluded();
excluded = filters.get(0).serviceParamsExcluded();
} else {
Set<String> includedList = new HashSet<>();
Set<String> excludedList = new HashSet<>();
for (MetadataParamsFilter filter : filters) {
if (ArrayUtils.isNotEmpty(filter.serviceParamsIncluded())) {
includedList.addAll(Arrays.asList(filter.serviceParamsIncluded()));
}
if (ArrayUtils.isNotEmpty(filter.serviceParamsExcluded())) {
excludedList.addAll(Arrays.asList(filter.serviceParamsExcluded()));
}
}
included = includedList.toArray(new String[0]);
excluded = excludedList.toArray(new String[0]);
}

if (ArrayUtils.isNotEmpty(included)) {
String[] methods = url.getParameter(METHODS_KEY, (String[]) null);
for (String p : included) {
String value = url.getParameter(p);
if (StringUtils.isNotEmpty(value) && params.get(p) == null) {
params.put(p, value);
}
appendMethodParams(url, params, methods, p);
}
} else if (ArrayUtils.isNotEmpty(excluded)) {
for (Map.Entry<String, String> entry : url.getParameters().entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
boolean shouldAdd = true;
for (String excludeKey : excluded) {
if (key.equalsIgnoreCase(excludeKey) || key.contains("." + excludeKey)) {
shouldAdd = false;
break;
}
}
if (shouldAdd) {
params.put(key, value);
}
}
}

this.params = params;
return params;
}

private void appendMethodParams(URL url, Map<String, String> params, String[] methods, String p) {
if (methods != null) {
for (String method : methods) {
String mValue = url.getMethodParameterStrict(method, p);
if (StringUtils.isNotEmpty(mValue)) {
params.put(method + DOT_SEPARATOR + p, mValue);
}
}
}
}

/**
* Initialize necessary caches right after deserialization on the consumer side
*/
@@ -18,6 +18,17 @@

import org.apache.dubbo.common.extension.SPI;

/**
* This filter applies an either 'include' or 'exclude' policy with 'include' having higher priority.
* That means if 'include' is specified then params specified in 'exclude' will be ignored
*
* If multiple Filter extensions are provided, then,
* 1. All params specified as should be included within different Filter extension instances will determine the params that will finally be used.
* 2. If none of the Filter extensions specified any params as should be included, then the final effective params would be those left after removed all the params specified as should be excluded.
*
* It is recommended for most users to use 'exclude' policy for service params and 'include' policy for instance params.
* Please use 'params-filter=-default, -filterName1, filterName2' to activate or deactivate filter extensions.
*/
@SPI
public interface MetadataParamsFilter {

@@ -26,12 +37,34 @@ public interface MetadataParamsFilter {
*
* @return arrays of keys
*/
String[] serviceParamsIncluded();
default String[] serviceParamsIncluded() {
return new String[0];
}

/**
* params that need to be sent to registry center
*
* @return arrays of keys
*/
String[] instanceParamsIncluded();
/**
* params that need to be excluded before sending to metadata center
*
* @return arrays of keys
*/
default String[] serviceParamsExcluded() {
return new String[0];
}

/**
* params that need to be sent to registry center
*
* @return arrays of keys
*/
default String[] instanceParamsIncluded() {
return new String[0];
}

/**
* params that need to be excluded before sending to registry center
*
* @return arrays of keys
*/
default String[] instanceParamsExcluded() {
return new String[0];
}
}
@@ -0,0 +1 @@
dubbo=org.apache.dubbo.metadata.DefaultMetadataParamsFilter
@@ -51,6 +51,12 @@ public class MetadataInfoTest {
"&metadata-type=remote&methods=sayHello&pid=36621&release=&revision=1.0.0&service-name-mapping=true" +
"&side=provider&timeout=5000&timestamp=1629970068002&version=1.0.0&params-filter=customized,-excluded");

private static URL url3 = URL.valueOf("dubbo://30.225.21.30:20880/org.apache.dubbo.registry.service.DemoService?" +
"REGISTRY_CLUSTER=registry1&anyhost=true&application=demo-provider2&delay=5000&deprecated=false&dubbo=2.0.2" +
"&dynamic=true&generic=false&group=greeting&interface=org.apache.dubbo.registry.service.DemoService" +
"&metadata-type=remote&methods=sayHello&sayHello.timeout=7000&pid=36621&release=&revision=1.0.0&service-name-mapping=true" +
"&side=provider&timeout=5000&timestamp=1629970068002&version=1.0.0&params-filter=-customized,excluded");

@Test
public void testEmptyRevision() {
MetadataInfo metadataInfo = new MetadataInfo("demo");
@@ -60,7 +66,7 @@ public void testEmptyRevision() {
}

@Test
public void testParamsFiltered() {
public void testParamsFilterIncluded() {
MetadataInfo metadataInfo = new MetadataInfo("demo");

// export normal url again
@@ -77,6 +83,24 @@ public void testParamsFiltered() {
assertEquals("7000", serviceInfo2.getMethodParameter("sayHello", TIMEOUT_KEY, "1000"));
}

@Test
public void testParamsFilterExcluded() {
MetadataInfo metadataInfo = new MetadataInfo("demo");

// export normal url again
metadataInfo.addService(url3);
MetadataInfo.ServiceInfo serviceInfo3 = metadataInfo.getServiceInfo(url3.getProtocolServiceKey());
assertNotNull(serviceInfo3);
assertEquals(14, serviceInfo3.getParams().size());
assertNotNull(serviceInfo3.getParams().get(INTERFACE_KEY));
assertNotNull(serviceInfo3.getParams().get(APPLICATION_KEY));
assertNotNull(serviceInfo3.getParams().get(VERSION_KEY));
assertNull(serviceInfo3.getParams().get(GROUP_KEY));
assertNull(serviceInfo3.getParams().get(TIMEOUT_KEY));
assertNull(serviceInfo3.getParams().get("anyhost"));
assertEquals("1000", serviceInfo3.getMethodParameter("sayHello", TIMEOUT_KEY, "1000"));
}

@Test
public void testEqualsAndRevision() {
// same metadata

0 comments on commit fe2d7dc

Please sign in to comment.