Skip to content

[azure] Add new source types and other cleanup #354

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 28, 2020
Merged

[azure] Add new source types and other cleanup #354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
032ada7
inital commit for new sources
claudiadadamo Sep 22, 2020
aa55a87
update tests and replace allSettled
claudiadadamo Sep 22, 2020
18f12d8
make empty string checking a bit more clear
claudiadadamo Sep 22, 2020
76dd26e
handle trailing slash
claudiadadamo Sep 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 43 additions & 24 deletions azure/activity_logs_monitoring/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ class EventhubLogForwarder {
}

addTagsToJsonLog(record) {
var metadata = this.extractResourceId(record);
var metadata = this.extractMetadataFromResource(record);
record['ddsource'] = metadata.source || DD_SOURCE;
record['ddsourcecategory'] = DD_SOURCE_CATEGORY;
record['service'] = DD_SERVICE;
Expand All @@ -225,39 +225,58 @@ class EventhubLogForwarder {
return this.addTagsToJsonLog(jsonLog);
}

extractResourceId(record) {
formatSourceType(sourceType) {
if (sourceType.includes('microsoft.')) {
return sourceType.replace('microsoft.', 'azure.');
} else {
return '';
}
}

extractMetadataFromResource(record) {
var metadata = { tags: [], source: '' };
if (
record.resourceId === undefined ||
typeof record.resourceId !== 'string'
) {
return metadata;
} else if (
record.resourceId.toLowerCase().startsWith('/subscriptions/')
) {
var resourceId = record.resourceId.toLowerCase().split('/');
if (resourceId.length > 2) {
metadata.tags.push('subscription_id:' + resourceId[2]);
}
var resourceId = record.resourceId.toLowerCase().split('/');
if (resourceId[0] === '') {
Copy link

@SamGRosen SamGRosen Sep 22, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what format the resource id's are, but this could have unexpected behavior if the string has the form "// multiple slashes".

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We really only expect the resource to come in like starting with a slash, and splitting by slash will give us an empty string as the first item. I made it conditional in case for some reason azure sends us something without a slash but I don't think that's likely, and i don't think it would be likely that they'd send something with multiple slashes.

resourceId = resourceId.slice(1);
}
if (resourceId[resourceId.length - 1] === '') {
resourceId.pop();
}

if (resourceId[0] === 'subscriptions') {
if (resourceId.length > 1) {
metadata.tags.push('subscription_id:' + resourceId[1]);
if (resourceId.length == 2) {
metadata.source = 'azure.subscription';
return metadata;
}
}
if (resourceId.length > 4) {
metadata.tags.push('resource_group:' + resourceId[4]);
if (resourceId.length > 3) {
metadata.tags.push('resource_group:' + resourceId[3]);
if (resourceId.length == 4) {
metadata.source = 'azure.resourcegroup';
return metadata;
}
}
if (resourceId.length > 6 && resourceId[6]) {
metadata.source = resourceId[6].replace('microsoft.', 'azure.');
if (resourceId.length > 5 && resourceId[5]) {
metadata.source = this.formatSourceType(resourceId[5]);
}
return metadata;
} else if (record.resourceId.toLowerCase().startsWith('/tenants/')) {
var resourceId = record.resourceId.toLowerCase().split('/');
if (resourceId.length > 4 && resourceId[4]) {
metadata.tags.push('tenant:' + resourceId[2]);
metadata.source = resourceId[4]
.replace('microsoft.', 'azure.')
.replace('aadiam', 'activedirectory');
} else if (resourceId[0] === 'tenants') {
if (resourceId.length > 3 && resourceId[3]) {
metadata.tags.push('tenant:' + resourceId[1]);
metadata.source = this.formatSourceType(resourceId[3]).replace(
'aadiam',
'activedirectory'
);
}
return metadata;
} else {
return metadata;
}
return metadata;
}
}

Expand All @@ -272,7 +291,7 @@ module.exports = async function(context, eventHubMessages) {
eventHubMessages
);

return Promise.allSettled(promises);
return Promise.all(promises.map(p => p.catch(e => e)));
};

module.exports.forTests = {
Expand Down
136 changes: 122 additions & 14 deletions azure/test/client.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ describe('Azure Log Monitoring', function() {
});
});

describe('#extractResourceId', function() {
describe('#extractMetadataFromResource', function() {
beforeEach(function() {
this.forwarder = setUp();
});
Expand All @@ -110,14 +110,48 @@ describe('Azure Log Monitoring', function() {
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid record without provider', function() {
it('should parse a valid resource group resource', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP'
};
expectedMetadata = {
tags: [
'subscription_id:12345678-1234-abcd-1234-1234567890ab',
'resource_group:some-resource-group'
],
source: 'azure.resourcegroup'
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid resource group resource ending slash', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/'
};
expectedMetadata = {
tags: [
'subscription_id:12345678-1234-abcd-1234-1234567890ab',
'resource_group:some-resource-group'
],
source: 'azure.resourcegroup'
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid record without provider length 5', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/ffffff'
};
expectedMetadata = {
tags: [
'subscription_id:12345678-1234-abcd-1234-1234567890ab',
Expand All @@ -127,45 +161,73 @@ describe('Azure Log Monitoring', function() {
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid record without provider and resource group', function() {
it('should parse a valid subscription type resource', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB'
};
expectedMetadata = {
tags: ['subscription_id:12345678-1234-abcd-1234-1234567890ab'],
source: 'azure.subscription'
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid subscription type resource ending slash', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/'
};
expectedMetadata = {
tags: ['subscription_id:12345678-1234-abcd-1234-1234567890ab'],
source: 'azure.subscription'
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
it('should parse a valid record without provider and resource group length 3', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/ffffff'
};
expectedMetadata = {
tags: ['subscription_id:12345678-1234-abcd-1234-1234567890ab'],
source: ''
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should not fail on record without resourceId', function() {
record = { key: 'value' };
expectedMetadata = { tags: [], source: '' };
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should not fail on string record', function() {
record = { key: 'value' };
expectedMetadata = { tags: [], source: '' };
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should not fail on improper resourceId', function() {
record = { resourceId: 'foo/bar' };
expectedMetadata = { tags: [], source: '' };
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should not fail with an invalid source', function() {
Expand All @@ -182,7 +244,41 @@ describe('Azure Log Monitoring', function() {
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractResourceId(record)
this.forwarder.extractMetadataFromResource(record)
);
});
it('should return empty source when not correct source format', function() {
record = {
resourceId:
'/SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/NOTTHESAMEFORMAT/VIRTUALMACHINES/SOME-VM'
};
expectedMetadata = {
tags: [
'subscription_id:12345678-1234-abcd-1234-1234567890ab',
'resource_group:some-resource-group'
],
source: ''
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
it('should handle when first element of resource id list is not empty', function() {
record = {
resourceId:
'SUBSCRIPTIONS/12345678-1234-ABCD-1234-1234567890AB/RESOURCEGROUPS/SOME-RESOURCE-GROUP/PROVIDERS/NOTTHESAMEFORMAT/VIRTUALMACHINES/SOME-VM'
};
expectedMetadata = {
tags: [
'subscription_id:12345678-1234-abcd-1234-1234567890ab',
'resource_group:some-resource-group'
],
source: ''
};
assert.deepEqual(
expectedMetadata,
this.forwarder.extractMetadataFromResource(record)
);
});
});
Expand All @@ -209,10 +305,7 @@ describe('Azure Log Monitoring', function() {
it('should handle string properly', function() {
log = 'hello';
expected = ['hello'];
assert.equal(
this.forwarder.getLogFormat(log),
constants.STRING
);
assert.equal(this.forwarder.getLogFormat(log), constants.STRING);
testHandleStringLogs(this.forwarder, log, expected);
});

Expand Down Expand Up @@ -328,4 +421,19 @@ describe('Azure Log Monitoring', function() {
testHandleStringLogs(this.forwarder, log, expected);
});
});
describe('#formatSourceType', function() {
beforeEach(function() {
this.forwarder = setUp();
});
it('should replace microsoft with azure', function() {
expected = 'azure.bleh';
actual = this.forwarder.formatSourceType('microsoft.bleh');
assert.equal(actual, expected);
});
it('should return empty source', function() {
expected = '';
actual = this.forwarder.formatSourceType('something');
assert.equal(actual, expected);
});
});
});