Skip to content

Commit

Permalink
remove string usage in metadata
Browse files Browse the repository at this point in the history 
Related to #358

This removes all usage of strings as metadata keys in Java code. This
does not yet cover off the relationship with cloud functions, which
would be the next step.

What also needs to happen following this is a larger consolidation of
key names as there are a lot of duplicates and potentially unused
values, and we can probably reduce the total key count.
  • Loading branch information
Aaron Meihm committed Apr 15, 2020
1 parent 097a431 commit c71d1d7
Show file tree
Hide file tree
Showing 58 changed files with 1,350 additions and 1,261 deletions.
15 changes: 5 additions & 10 deletions src/main/java/com/mozilla/secops/IprepdIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mozilla.secops.alert.Alert;
import com.mozilla.secops.alert.AlertMeta;
import com.mozilla.secops.crypto.RuntimeSecrets;
import com.mozilla.secops.state.DatastoreStateInterface;
import com.mozilla.secops.state.State;
Expand Down Expand Up @@ -35,12 +36,6 @@

/** {@link IprepdIO} provides an IO transform for writing violation messages to iprepd */
public class IprepdIO {
/** Metadata tag in an alert to indicate iprepd exemption */
public static final String IPREPD_EXEMPT = "iprepd_exempt";

/** Metadata tag in an alert to indicate recovery suppression */
public static final String IPREPD_SUPPRESS_RECOVERY = "iprepd_suppress_recovery";

/** Custom metric name used to count iprepd violation submissions from write functions */
public static final String VIOLATION_WRITES_METRIC = "iprepd_violation_writes";

Expand Down Expand Up @@ -387,7 +382,7 @@ public void processElement(ProcessContext c) {
return;
}

String iprepdExempt = a.getMetadataValue(IPREPD_EXEMPT);
String iprepdExempt = a.getMetadataValue(AlertMeta.Key.IPREPD_EXEMPT);
if (iprepdExempt != null && iprepdExempt.equals("true")) {
return;
}
Expand Down Expand Up @@ -473,7 +468,7 @@ public void processElement(ProcessContext c) {
* @param a Alert
*/
public static void addMetadataSuppressRecovery(Integer value, Alert a) {
a.addMetadata(IPREPD_SUPPRESS_RECOVERY, value.toString());
a.addMetadata(AlertMeta.Key.IPREPD_SUPPRESS_RECOVERY, value.toString());
}

/** WhitelistedObject contains the metadata associated with a whitelisted objects. */
Expand Down Expand Up @@ -677,8 +672,8 @@ public static void addMetadataIfObjectWhitelisted(
sc = state.newCursor(WhitelistedObject.class, false);
WhitelistedObject wobj = sc.get(obj);
if (wobj != null) {
a.addMetadata(IPREPD_EXEMPT, "true");
a.addMetadata(IPREPD_EXEMPT + "_created_by", wobj.getCreatedBy());
a.addMetadata(AlertMeta.Key.IPREPD_EXEMPT, "true");
a.addMetadata(AlertMeta.Key.IPREPD_EXEMPT_CREATED_BY, wobj.getCreatedBy());
}
} catch (StateException exc) {
log.error("error getting whitelisted object: {}", exc.getMessage());
Expand Down
6 changes: 0 additions & 6 deletions src/main/java/com/mozilla/secops/OutputOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,6 @@ public interface OutputOptions extends PipelineOptions, GcpOptions {
@Hidden
void setOutputAlertTemplates(String[] value);

@Description(
"Adds GeoIP metadata to all fields specified (multiple allowed); keys names for alert metadata with an IP address, for example: sourceaddress")
String[] getAlertAddressFields();

void setAlertAddressFields(String[] value);

@Description(
"Email address to receive critical alert notifications if application generates them")
String getCriticalNotificationEmail();
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/com/mozilla/secops/SourceCorrelation.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.mozilla.secops;

import com.mozilla.secops.alert.Alert;
import com.mozilla.secops.alert.AlertMeta;
import com.mozilla.secops.httprequest.HTTPRequestToggles;
import com.mozilla.secops.parser.Event;
import com.mozilla.secops.parser.Normalized;
Expand Down Expand Up @@ -282,9 +283,9 @@ public void processElement(ProcessContext c) {
monitoredResource, isp, alert, cnt));
a.setCategory("httprequest");
a.setSubcategory("isp_source_correlation");
a.addMetadata("sourceaddress_isp", isp);
a.addMetadata("total_address_count", Integer.toString(cnt));
a.addMetadata("total_alert_count", Integer.toString(alert));
a.addMetadata(AlertMeta.Key.SOURCEADDRESS_ISP, isp);
a.addMetadata(AlertMeta.Key.TOTAL_ADDRESS_COUNT, Integer.toString(cnt));
a.addMetadata(AlertMeta.Key.TOTAL_ALERT_COUNT, Integer.toString(alert));
a.setNotifyMergeKey(
String.format("%s isp_source_correlation", monitoredResource));
c.output(a);
Expand Down Expand Up @@ -330,15 +331,15 @@ public static class AlertSourceExtractor extends DoFn<Alert, SourceData> {
@ProcessElement
public void processElement(ProcessContext c) {
Alert a = c.element();
if (a.getMetadataValue("sourceaddress") == null) {
if (a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS) == null) {
return;
}
SourceData o = new SourceData();
o.setSourceAddress(a.getMetadataValue("sourceaddress"));
o.setCity(a.getMetadataValue("sourceaddress_city"));
o.setCountry(a.getMetadataValue("sourceaddress_country"));
o.setIsp(a.getMetadataValue("sourceaddress_isp"));
String asns = a.getMetadataValue("sourceaddress_asn");
o.setSourceAddress(a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS));
o.setCity(a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS_CITY));
o.setCountry(a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS_COUNTRY));
o.setIsp(a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS_ISP));
String asns = a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS_ASN);
if (asns != null) {
o.setAsn(new Integer(asns));
}
Expand Down
19 changes: 11 additions & 8 deletions src/main/java/com/mozilla/secops/Violation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mozilla.secops.alert.Alert;
import com.mozilla.secops.alert.AlertMeta;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -67,7 +68,7 @@ private abstract static class ViolationGenerator {
public abstract Violation[] generate(Alert a);

protected Violation createViolation(Alert a, String object, String type, String vStr) {
String suppressValue = a.getMetadataValue(IprepdIO.IPREPD_SUPPRESS_RECOVERY);
String suppressValue = a.getMetadataValue(AlertMeta.Key.IPREPD_SUPPRESS_RECOVERY);
if (suppressValue == null) {
return new Violation(object, type, vStr);
} else {
Expand All @@ -81,10 +82,12 @@ private static class GenericSourceViolationGenerator extends ViolationGenerator

public Violation[] generate(Alert a) {
ArrayList<Violation> ret = new ArrayList<>();
if (a.getMetadataValue("sourceaddress") == null) {
if (a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS) == null) {
return null;
}
ret.add(createViolation(a, a.getMetadataValue("sourceaddress"), "ip", vType.toString()));
ret.add(
createViolation(
a, a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS), "ip", vType.toString()));
return ret.toArray(new Violation[ret.size()]);
}

Expand All @@ -98,7 +101,7 @@ private static class EmailListViolationGenerator extends ViolationGenerator {

public Violation[] generate(Alert a) {
ArrayList<Violation> ret = new ArrayList<>();
String emails = a.getMetadataValue("email");
String emails = a.getMetadataValue(AlertMeta.Key.EMAIL);
if (emails == null) {
return null;
}
Expand All @@ -119,17 +122,17 @@ public Violation[] generate(Alert a) {
// Custom generator implementation; we will create a violation both for the email
// address if present and the source address.
ArrayList<Violation> ret = new ArrayList<>();
if (a.getMetadataValue("sourceaddress") == null) {
if (a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS) == null) {
return null;
}
ret.add(
createViolation(
a,
a.getMetadataValue("sourceaddress"),
a.getMetadataValue(AlertMeta.Key.SOURCEADDRESS),
"ip",
ViolationType.ENDPOINT_ABUSE_VIOLATION.toString()));
if (a.getMetadataValue("email") != null) {
String[] parts = a.getMetadataValue("email").split(", ?");
if (a.getMetadataValue(AlertMeta.Key.EMAIL) != null) {
String[] parts = a.getMetadataValue(AlertMeta.Key.EMAIL).split(", ?");
for (String i : parts) {
ret.add(
createViolation(a, i, "email", ViolationType.ABUSIVE_ACCOUNT_VIOLATION.toString()));
Expand Down
81 changes: 50 additions & 31 deletions src/main/java/com/mozilla/secops/alert/Alert.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
public class Alert implements Serializable {
private static final long serialVersionUID = 1L;

public static final String ALERT_SUBCATEGORY_FIELD = "category";

public enum AlertSeverity {
/** Informational */
@JsonProperty("info")
Expand Down Expand Up @@ -126,7 +124,7 @@ public String getSummary() {
* @param maskedSummary Masked summary string
*/
public void setMaskedSummary(String maskedSummary) {
addMetadata("masked_summary", maskedSummary);
addMetadata(AlertMeta.Key.MASKED_SUMMARY, maskedSummary);
}

/**
Expand All @@ -136,7 +134,7 @@ public void setMaskedSummary(String maskedSummary) {
*/
@JsonIgnore
public String getMaskedSummary() {
return getMetadataValue("masked_summary");
return getMetadataValue(AlertMeta.Key.MASKED_SUMMARY);
}

/**
Expand All @@ -149,7 +147,7 @@ public String getMaskedSummary() {
* @param key Merge key for alert notifications
*/
public void setNotifyMergeKey(String key) {
addMetadata("notify_merge", key);
addMetadata(AlertMeta.Key.NOTIFY_MERGE, key);
}

/**
Expand All @@ -159,7 +157,7 @@ public void setNotifyMergeKey(String key) {
*/
@JsonIgnore
public String getNotifyMergeKey() {
return getMetadataValue("notify_merge");
return getMetadataValue(AlertMeta.Key.NOTIFY_MERGE);
}

/**
Expand Down Expand Up @@ -209,7 +207,25 @@ public String getPayload() {
* @param key Key to return data for
* @return Value string, null if not found
*/
public String getMetadataValue(String key) {
public String getMetadataValue(AlertMeta.Key key) {
for (AlertMeta m : metadata) {
if (m.getKey().equals(key.getKey())) {
return m.getValue();
}
}
return null;
}

/**
* Return a custom metadata value
*
* <p>Custom metadata includes arbitrary keys that are not standardized, such as keys set in
* configuration ticks. This method should not be used under normal circumstances.
*
* @param key Key to return data for
* @return Value string, null if not found
*/
public String getCustomMetadataValue(String key) {
for (AlertMeta m : metadata) {
if (m.getKey().equals(key)) {
return m.getValue();
Expand All @@ -227,16 +243,17 @@ public String getMetadataValue(String key) {
* @param key Key to set
* @param value Value to set for key
*/
public void setMetadataValue(String key, String value) {
public void setMetadataValue(AlertMeta.Key key, String value) {
key.validate(value);
metaLock.lock();
try {
for (AlertMeta m : metadata) {
if (m.getKey().equals(key)) {
if (m.getKey().equals(key.getKey())) {
m.setValue(value);
return;
}
}
metadata.add(new AlertMeta(key, value));
metadata.add(new AlertMeta(key.getKey(), value));
} finally {
metaLock.unlock();
}
Expand All @@ -262,7 +279,6 @@ public ArrayList<AlertMeta> getMetadata() {
@JsonProperty("metadata")
public void setMetadata(ArrayList<AlertMeta> metadata) {
this.metadata = metadata;
AlertCompat.compatibility(this);
}

/**
Expand All @@ -271,34 +287,38 @@ public void setMetadata(ArrayList<AlertMeta> metadata) {
* @param key Key
* @param value Value
*/
public void addMetadata(String key, String value) {
public void addMetadata(AlertMeta.Key key, String value) {
key.validate(value);
// Pick up metadata mutex here to prevent ConcurrentModification exception if object is
// serialized while we are appending to the metadata, see local implementation of
// writeObject
metaLock.lock();
try {
metadata.add(new AlertMeta(key, value));
metadata.add(new AlertMeta(key.getKey(), value));
} finally {
metaLock.unlock();
}
}

/**
* A best effort addMetadata(k,v)
* Set a custom metadata value
*
* <p>If a null or empty key or value is provided, this function will not add anything. Do NOT use
* this function for metadata entries that are not purely informational.
* <p>Custom metadata includes arbitrary keys that are not standardized, such as keys set in
* configuration ticks. This method should not be used under normal circumstances.
*
* @param k String key
* @param v String value
* @return true if metadata entry was set, false otherwise
* @param key Key to set
* @param value Value to set
*/
public boolean tryAddMetadata(String k, String v) {
if (k == null || v == null || k.equals("") || v.equals("")) {
return false;
public void addCustomMetadata(String key, String value) {
// Pick up metadata mutex here to prevent ConcurrentModification exception if object is
// serialized while we are appending to the metadata, see local implementation of
// writeObject
metaLock.lock();
try {
metadata.add(new AlertMeta(key, value));
} finally {
metaLock.unlock();
}
addMetadata(k, v);
return true;
}

/**
Expand Down Expand Up @@ -346,7 +366,7 @@ public String getCategory() {
*/
@JsonIgnore
public String getSubcategory() {
return getMetadataValue(ALERT_SUBCATEGORY_FIELD);
return getMetadataValue(AlertMeta.Key.ALERT_SUBCATEGORY_FIELD);
}

/**
Expand All @@ -360,8 +380,7 @@ public void setSubcategory(String subcategory) {
throw new IllegalArgumentException("attempt to set subcategory with no category");
}

addMetadata(ALERT_SUBCATEGORY_FIELD, subcategory);
AlertCompat.compatibility(this);
addMetadata(AlertMeta.Key.ALERT_SUBCATEGORY_FIELD, subcategory);
}

/**
Expand All @@ -370,7 +389,7 @@ public void setSubcategory(String subcategory) {
* @param templateName Freemarker template name with file extension
*/
public void setEmailTemplate(String templateName) {
addMetadata("template_name_email", templateName);
addMetadata(AlertMeta.Key.TEMPLATE_NAME_EMAIL, templateName);
}

/**
Expand All @@ -380,7 +399,7 @@ public void setEmailTemplate(String templateName) {
*/
@JsonIgnore
public String getEmailTemplate() {
return getMetadataValue("template_name_email");
return getMetadataValue(AlertMeta.Key.TEMPLATE_NAME_EMAIL);
}

/**
Expand All @@ -389,7 +408,7 @@ public String getEmailTemplate() {
* @param templateName Freemarker template name with file extension
*/
public void setSlackTemplate(String templateName) {
addMetadata("template_name_slack", templateName);
addMetadata(AlertMeta.Key.TEMPLATE_NAME_SLACK, templateName);
}

/**
Expand All @@ -399,7 +418,7 @@ public void setSlackTemplate(String templateName) {
*/
@JsonIgnore
public String getSlackTemplate() {
return getMetadataValue("template_name_slack");
return getMetadataValue(AlertMeta.Key.TEMPLATE_NAME_SLACK);
}

/**
Expand Down

0 comments on commit c71d1d7

Please sign in to comment.