Skip to content

Tc.v2.4.5 fix #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: tc.v2.4.5
Choose a base branch
from
Open

Tc.v2.4.5 fix #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b202ad9
- fix tabs
dancosta93 Jul 31, 2020
52bf664
- Fix bug when having more than 1 page of results. The BlobPageFetche…
dancosta93 Aug 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 48 additions & 44 deletions plugins/cloud-gce/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@ governing permissions and limitations under the License. -->

<artifactId>cloud-gce</artifactId>
<name>Plugin: Cloud: Google Compute Engine</name>
<description>The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism.</description>
<description>The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery
mechanism.
</description>

<properties>
<elasticsearch.plugin.classname>org.elasticsearch.plugin.cloud.gce.CloudGcePlugin</elasticsearch.plugin.classname>
<elasticsearch.plugin.classname>org.elasticsearch.plugin.cloud.gce.CloudGcePlugin
</elasticsearch.plugin.classname>
<google.gce.version>v1-rev144-1.21.0</google.gce.version>
<google.gcs.version>1.0.0</google.gcs.version>
<google.gcs.version>1.0.0</google.gcs.version>
<!-- currently has no unit tests -->
<tests.rest.suite>cloud_gce</tests.rest.suite>
<tests.rest.load_packaged>false</tests.rest.load_packaged>
Expand All @@ -39,21 +42,21 @@ governing permissions and limitations under the License. -->
<dependencies>
<!-- Google APIs -->
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-compute</artifactId>
<version>${google.gce.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-compute</artifactId>
<version>${google.gce.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava-jdk5</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>${google.gcs.version}</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>${google.gcs.version}</version>
</dependency>
<!-- We need to force here the compile scope as it was defined as test scope in plugins/pom.xml -->
<!-- TODO: remove this dependency when we will have a REST Test module -->
<dependency>
Expand All @@ -65,7 +68,7 @@ governing permissions and limitations under the License. -->

<build>
<plugins>
<plugin>
<plugin>
<groupId>de.thetaphi</groupId>
<artifactId>forbiddenapis</artifactId>
<executions>
Expand Down Expand Up @@ -117,8 +120,8 @@ governing permissions and limitations under the License. -->
<configuration>
<target>
<!-- if we are on java 1.8.* we set doclint to run all checks -->
<condition property="doclint.all" value="-Xdoclint:all" else="" >
<matches pattern="1\.8\..+$" string="${java.runtime.version}" />
<condition property="doclint.all" value="-Xdoclint:all" else="">
<matches pattern="1\.8\..+$" string="${java.runtime.version}"/>
</condition>
</target>
<exportAntProperties>true</exportAntProperties>
Expand All @@ -133,8 +136,8 @@ governing permissions and limitations under the License. -->
<configuration>
<target>
<!-- if we are on java 1.8.* we set doclint to ignore missing-->
<condition property="doclint.missing" value="-Xdoclint:-missing" else="" >
<matches pattern="1\.8\..+$" string="${java.runtime.version}" />
<condition property="doclint.missing" value="-Xdoclint:-missing" else="">
<matches pattern="1\.8\..+$" string="${java.runtime.version}"/>
</condition>
</target>
<exportAntProperties>true</exportAntProperties>
Expand Down Expand Up @@ -165,28 +168,29 @@ governing permissions and limitations under the License. -->
<artifactId>maven-assembly-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<!-- generate certificates/keys -->
<execution>
<id>certificate-setup</id>
<phase>generate-test-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<delete file="${keystore}" quiet="true"/> <!-- must clean it up first -->
<mkdir dir="${project.build.testOutputDirectory}"/> <!-- the directory must exist first -->
<exec executable="keytool" failonerror="true">
<arg line="-genkey -keyalg RSA -alias selfsigned -keystore ${keystore} -storepass keypass -keypass keypass -validity 712 -keysize 2048 -dname CN=127.0.0.1"/>
</exec>
</target>
<skip>${skip.unit.tests}</skip> <!-- the GceDiscoverTests run as a unittest but are really integ tests that's why we skip this task if unittests don't run -->
</configuration>
</execution>
</executions>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<!-- generate certificates/keys -->
<execution>
<id>certificate-setup</id>
<phase>generate-test-resources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<delete file="${keystore}" quiet="true"/> <!-- must clean it up first -->
<mkdir dir="${project.build.testOutputDirectory}"/> <!-- the directory must exist first -->
<exec executable="keytool" failonerror="true">
<arg line="-genkey -keyalg RSA -alias selfsigned -keystore ${keystore} -storepass keypass -keypass keypass -validity 712 -keysize 2048 -dname CN=127.0.0.1"/>
</exec>
</target>
<skip>${skip.unit.tests}
</skip> <!-- the GceDiscoverTests run as a unittest but are really integ tests that's why we skip this task if unittests don't run -->
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
Expand Down
4 changes: 3 additions & 1 deletion plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ final class Fields {

/**
* Return a collection of running instances within the same GCE project
*
* @return a collection of running instances within the same GCE project
*/
Collection<Instance> instances();
Expand All @@ -54,10 +55,11 @@ final class Fields {
* <li>`hostname` when we need to resolve the host name</li>
* <li>`network-interfaces/0/ip` when we need to resolve private IP</li>
* </ul>
* @see org.elasticsearch.cloud.gce.network.GceNameResolver for bindings
*
* @param metadataPath path to metadata information
* @return extracted information (for example a hostname or an IP address)
* @throws IOException in case metadata URL is not accessible
* @see org.elasticsearch.cloud.gce.network.GceNameResolver for bindings
*/
String metadata(String metadataPath) throws IOException;
}
96 changes: 51 additions & 45 deletions plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import java.util.List;

import static org.elasticsearch.common.util.CollectionUtils.eagerTransform;

import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import java.util.*;
Expand Down Expand Up @@ -83,47 +84,47 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute

@Override
public Collection<Instance> instances() {
logger.debug("get instances for project [{}], zones [{}]", project, zones);

List<List<Instance>> instanceListByZone = eagerTransform(zones, new Function<String, List<Instance>>() {
@Override
public List<Instance> apply(final String zoneId) {
try {
// hack around code messiness in GCE code
// TODO: get this fixed
SecurityManager sm = System.getSecurityManager();
if (sm != null) {
sm.checkPermission(new SpecialPermission());
}
InstanceList instanceList = AccessController.doPrivileged(new PrivilegedExceptionAction<InstanceList>() {
@Override
public InstanceList run() throws Exception {
Compute.Instances.List list = client().instances().list(project, zoneId);
return list.execute();
}
});
if (instanceList.isEmpty() || instanceList.getItems() == null) {
return Collections.EMPTY_LIST;
logger.debug("get instances for project [{}], zones [{}]", project, zones);

List<List<Instance>> instanceListByZone = eagerTransform(zones, new Function<String, List<Instance>>() {
@Override
public List<Instance> apply(final String zoneId) {
try {
// hack around code messiness in GCE code
// TODO: get this fixed
SecurityManager sm = System.getSecurityManager();
if (sm != null) {
sm.checkPermission(new SpecialPermission());
}
InstanceList instanceList = AccessController.doPrivileged(new PrivilegedExceptionAction<InstanceList>() {
@Override
public InstanceList run() throws Exception {
Compute.Instances.List list = client().instances().list(project, zoneId);
return list.execute();
}

return instanceList.getItems();
} catch (PrivilegedActionException e) {
logger.warn("Problem fetching instance list for zone {}", zoneId);
logger.debug("Full exception:", e);

});
if (instanceList.isEmpty() || instanceList.getItems() == null) {
return Collections.EMPTY_LIST;
}
}
});

// Collapse instances from all zones into one neat list
List<Instance> instanceList = CollectionUtils.iterableAsArrayList(Iterables.concat(instanceListByZone));
return instanceList.getItems();
} catch (PrivilegedActionException e) {
logger.warn("Problem fetching instance list for zone {}", zoneId);
logger.debug("Full exception:", e);

if (instanceList.size() == 0) {
logger.warn("disabling GCE discovery. Can not get list of nodes");
return Collections.EMPTY_LIST;
}
}
});

// Collapse instances from all zones into one neat list
List<Instance> instanceList = CollectionUtils.iterableAsArrayList(Iterables.concat(instanceListByZone));

if (instanceList.size() == 0) {
logger.warn("disabling GCE discovery. Can not get list of nodes");
}

return instanceList;
return instanceList;
}

@Override
Expand All @@ -146,9 +147,9 @@ public HttpHeaders run() throws IOException {
headers.put("Metadata-Flavor", "Google");
HttpResponse response;
response = getGceHttpTransport().createRequestFactory()
.buildGetRequest(new GenericUrl(url))
.setHeaders(headers)
.execute();
.buildGetRequest(new GenericUrl(url))
.setHeaders(headers)
.execute();
String metadata = response.parseAsString();
logger.debug("metadata found [{}]", metadata);
return metadata;
Expand All @@ -161,10 +162,14 @@ public HttpHeaders run() throws IOException {
private TimeValue refreshInterval = null;
private long lastRefresh;

/** Global instance of the HTTP transport. */
/**
* Global instance of the HTTP transport.
*/
private HttpTransport gceHttpTransport;

/** Global instance of the JSON factory. */
/**
* Global instance of the JSON factory.
*/
private JsonFactory gceJsonFactory;

@Inject
Expand All @@ -176,7 +181,7 @@ public GceComputeServiceImpl(Settings settings, NetworkService networkService) {
networkService.addCustomNameResolver(new GceNameResolver(settings, this));

this.gceHost = settings.get("cloud.gce.host", DEFAULT_GCE_HOST);
this.metaDataUrl = gceHost + "/computeMetadata/v1/instance";
this.metaDataUrl = gceHost + "/computeMetadata/v1/instance";
this.gceRootUrl = settings.get("cloud.gce.root_url", DEFAULT_GCE_ROOT_URL);
this.tokenServerEncodedUrl = metaDataUrl + "/service-accounts/default/token";
this.validateCerts = settings.getAsBoolean("cloud.gce.validate_certificates", true);
Expand All @@ -197,7 +202,7 @@ protected synchronized HttpTransport getGceHttpTransport() throws GeneralSecurit
public synchronized Compute client() {
if (refreshInterval != null && refreshInterval.millis() != 0) {
if (client != null &&
(refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) {
(refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) {
if (logger.isTraceEnabled()) logger.trace("using cache to retrieve client");
return client;
}
Expand All @@ -209,8 +214,8 @@ public synchronized Compute client() {

logger.info("starting GCE discovery service");
final ComputeCredential credential = new ComputeCredential.Builder(getGceHttpTransport(), gceJsonFactory)
.setTokenServerEncodedUrl(this.tokenServerEncodedUrl)
.build();
.setTokenServerEncodedUrl(this.tokenServerEncodedUrl)
.build();

// hack around code messiness in GCE code
// TODO: get this fixed
Expand All @@ -228,12 +233,13 @@ public Void run() throws IOException {

logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds());
if (credential.getExpiresInSeconds() != null) {
refreshInterval = TimeValue.timeValueSeconds(credential.getExpiresInSeconds()-1);
refreshInterval = TimeValue.timeValueSeconds(credential.getExpiresInSeconds() - 1);
}

final boolean ifRetry = settings.getAsBoolean(Fields.RETRY, true);
final Compute.Builder builder = new Compute.Builder(getGceHttpTransport(), gceJsonFactory, null)
.setApplicationName(Fields.VERSION).setRootUrl(gceRootUrl);;
.setApplicationName(Fields.VERSION).setRootUrl(gceRootUrl);
;

if (ifRetry) {
int maxWait = settings.getAsInt(Fields.MAXWAIT, -1);
Expand Down
4 changes: 2 additions & 2 deletions plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ private InetAddress[] resolve(String value) throws IOException {
gceMetadataPath = Strings.replace(GceAddressResolverType.PRIVATE_IP.gceName, "{{network}}", network);
} else {
throw new IllegalArgumentException("[" + value + "] is not one of the supported GCE network.host setting. " +
"Expecting _gce_, _gce:privateIp:X_, _gce:hostname_");
"Expecting _gce_, _gce:privateIp:X_, _gce:hostname_");
}

try {
Expand All @@ -110,7 +110,7 @@ private InetAddress[] resolve(String value) throws IOException {
throw new IOException("no gce metadata returned from [" + gceMetadataPath + "] for [" + value + "]");
}
// only one address: because we explicitly ask for only one via the GceHostnameType
return new InetAddress[] { InetAddress.getByName(metadataResult) };
return new InetAddress[]{InetAddress.getByName(metadataResult)};
} catch (IOException e) {
throw new IOException("IOException caught when fetching InetAddress from [" + gceMetadataPath + "]", e);
}
Expand Down
18 changes: 9 additions & 9 deletions plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gcs/GCSModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,16 @@
import org.elasticsearch.common.inject.AbstractModule;

public class GCSModule extends AbstractModule {

static Class<? extends GCSService> storageServiceImpl = GCSServiceImpl.class;

public static Class<? extends GCSService> getStorageServiceImpl() {
return storageServiceImpl;
static Class<? extends GCSService> storageServiceImpl = GCSServiceImpl.class;

public static Class<? extends GCSService> getStorageServiceImpl() {
return storageServiceImpl;
}

@Override
protected void configure() {
bind(GCSService.class).to(storageServiceImpl).asEagerSingleton();
}
@Override
protected void configure() {
bind(GCSService.class).to(storageServiceImpl).asEagerSingleton();
}

}
}
Loading