Skip to content
This repository has been archived by the owner on Mar 4, 2021. It is now read-only.

EddaInstanceJanitorCrawler: breakup edda queries for image ids #220

Merged
merged 1 commit into from
Nov 19, 2015
Merged

EddaInstanceJanitorCrawler: breakup edda queries for image ids #220

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 45 additions & 27 deletions ...main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaInstanceJanitorCrawler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
Expand All @@ -60,6 +61,8 @@ public class EddaInstanceJanitorCrawler implements JanitorCrawler {
private final List<String> regions = Lists.newArrayList();
private final Map<String, String> instanceToAsg = Maps.newHashMap();

/** Max image ids per Edda Query */
private static final int MAX_IMAGE_IDS_PER_QUERY = 40;

/**
* Instantiates a new basic instance crawler.
Expand Down Expand Up @@ -232,39 +235,54 @@ private void refreshAsgInstances() {
}

private void refreshOwnerByImage(String region, List<Resource> resources) {
HashSet<String> imageIds = new HashSet<String>();
HashSet<String> imageIds = new HashSet<>();
for (Resource resource: resources) {
if (resource.getOwnerEmail() == null) {
imageIds.add(resource.getAdditionalField("imageId"));
}
}
if (imageIds.size() > 0) {
HashMap<String, String> imageToOwner = new HashMap<String, String>();
String url = eddaClient.getBaseUrl(region) + "/aws/images/";
url += StringUtils.join(imageIds, ',');
url += String.format(";tags.key=%1$s;public=false;_expand:(imageId,tags:(%1$s))", BasicSimianArmyContext.GLOBAL_OWNER_TAGKEY);
JsonNode imageJsonNode = null;
try {
imageJsonNode = eddaClient.getJsonNodeFromUrl(url);
} catch (Exception e) {
LOGGER.error(String.format(
"Failed to get Json node from edda for AMIs in region %s.", region), e);
}
if (imageJsonNode == null) {
return;
}
for (Iterator<JsonNode> it = imageJsonNode.getElements(); it.hasNext();) {
JsonNode image = it.next();
String imageId = image.get("imageId").getTextValue();
JsonNode tags = image.get("tags");
for (Iterator<JsonNode> tagIt = tags.getElements(); tagIt.hasNext();) {
JsonNode tag = tagIt.next();
if (tag.get(BasicSimianArmyContext.GLOBAL_OWNER_TAGKEY) != null) {
imageToOwner.put(imageId, tag.get(BasicSimianArmyContext.GLOBAL_OWNER_TAGKEY).getTextValue());
break;
}
HashMap<String, String> imageToOwner = new HashMap<>();
String baseurl = eddaClient.getBaseUrl(region) + "/aws/images/";

Iterator<String> itr = imageIds.iterator();
long leftToQuery = imageIds.size();
while (leftToQuery > 0) {
long batchcount = leftToQuery > MAX_IMAGE_IDS_PER_QUERY ? MAX_IMAGE_IDS_PER_QUERY : leftToQuery;
leftToQuery -= batchcount;

ArrayList<String> batch = new ArrayList<>();
for(int i=0;i<batchcount; i++) {
batch.add(itr.next());
}
}

String url = baseurl;
url += StringUtils.join(batch, ',');
url += ";tags.key=owner;public=false;_expand:(imageId,tags:(owner))";
JsonNode imageJsonNode = null;
try {
imageJsonNode = eddaClient.getJsonNodeFromUrl(url);
} catch (Exception e) {
LOGGER.error(String.format(
"Failed to get Json node from edda for AMIs in region %s.", region), e);
}

if (imageJsonNode != null) {
for (Iterator<JsonNode> it = imageJsonNode.getElements(); it.hasNext();) {
JsonNode image = it.next();
String imageId = image.get("imageId").getTextValue();
JsonNode tags = image.get("tags");
for (Iterator<JsonNode> tagIt = tags.getElements(); tagIt.hasNext();) {
JsonNode tag = tagIt.next();
if (tag.get(BasicSimianArmyContext.GLOBAL_OWNER_TAGKEY) != null) {
imageToOwner.put(imageId, tag.get(BasicSimianArmyContext.GLOBAL_OWNER_TAGKEY).getTextValue());
break;
}
}
}
}
}

if (imageToOwner.size() > 0) {
for (Resource resource: resources) {
if (resource.getOwnerEmail() == null
Expand All @@ -275,6 +293,6 @@ private void refreshOwnerByImage(String region, List<Resource> resources) {
}
}
}
}
}
}
}