Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #18 from awslabs/cloudwatchlogs
Add cloudwatchlogs module and bumping version to 0.0.1! 🚀
- Loading branch information
Showing
22 changed files
with
623 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
__title__ = "awswrangler" | ||
__description__ = "Utility belt to handle data on AWS." | ||
__version__ = "0.0b32" | ||
__version__ = "0.0.1" | ||
__license__ = "Apache License 2.0" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
from time import sleep | ||
from datetime import datetime | ||
import logging | ||
|
||
from awswrangler.exceptions import QueryFailed, QueryCancelled | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
QUERY_WAIT_POLLING_DELAY = 0.2 # MILLISECONDS | ||
|
||
|
||
class CloudWatchLogs: | ||
def __init__(self, session): | ||
self._session = session | ||
self._client_logs = session.boto3_session.client( | ||
service_name="logs", config=session.botocore_config) | ||
|
||
def start_query(self, | ||
query, | ||
log_group_names, | ||
start_time=datetime(year=1970, month=1, day=1), | ||
end_time=datetime.utcnow(), | ||
limit=None): | ||
""" | ||
Run a query against AWS CloudWatchLogs Insights and wait the results | ||
https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_QuerySyntax.html | ||
:param query: The query string to use. | ||
:param log_group_names: The list of log groups to be queried. You can include up to 20 log groups. | ||
:param start_time: The beginning of the time range to query (datetime.datetime object) | ||
:param end_time: The end of the time range to query (datetime.datetime object) | ||
:param limit: The maximum number of log events to return in the query. | ||
:return: Query ID | ||
""" | ||
start_timestamp = int(1000 * start_time.timestamp()) | ||
end_timestamp = int(1000 * end_time.timestamp()) | ||
logger.debug(f"start_timestamp: {start_timestamp}") | ||
logger.debug(f"end_timestamp: {end_timestamp}") | ||
args = { | ||
"logGroupNames": log_group_names, | ||
"startTime": start_timestamp, | ||
"endTime": end_timestamp, | ||
"queryString": query | ||
} | ||
if limit: | ||
args["limit"] = limit | ||
response = self._client_logs.start_query(**args) | ||
return response["queryId"] | ||
|
||
def wait_query(self, query_id): | ||
""" | ||
Wait query ends | ||
:param query_id: Query ID | ||
:return: Query results | ||
""" | ||
final_states = ["Complete", "Failed", "Cancelled"] | ||
response = self._client_logs.get_query_results(queryId=query_id) | ||
status = response["status"] | ||
while status not in final_states: | ||
sleep(QUERY_WAIT_POLLING_DELAY) | ||
response = self._client_logs.get_query_results(queryId=query_id) | ||
status = response["status"] | ||
logger.debug(f"status: {status}") | ||
if status == "Failed": | ||
raise QueryFailed(f"query ID: {query_id}") | ||
elif status == "Cancelled": | ||
raise QueryCancelled(f"query ID: {query_id}") | ||
return response | ||
|
||
def query(self, | ||
query, | ||
log_group_names, | ||
start_time=datetime(year=1970, month=1, day=1), | ||
end_time=datetime.utcnow(), | ||
limit=None): | ||
""" | ||
Run a query against AWS CloudWatchLogs Insights and wait the results | ||
https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_QuerySyntax.html | ||
:param query: The query string to use. | ||
:param log_group_names: The list of log groups to be queried. You can include up to 20 log groups. | ||
:param start_time: The beginning of the time range to query (datetime.datetime object) | ||
:param end_time: The end of the time range to query (datetime.datetime object) | ||
:param limit: The maximum number of log events to return in the query. | ||
:return: Results | ||
""" | ||
query_id = self.start_query(query=query, | ||
log_group_names=log_group_names, | ||
start_time=start_time, | ||
end_time=end_time, | ||
limit=limit) | ||
response = self.wait_query(query_id=query_id) | ||
return response["results"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
FROM lambci/lambda:build-python3.6 | ||
FROM lambci/lambda:build-python3.7 | ||
|
||
RUN pip install --upgrade pip | ||
|
||
|
Oops, something went wrong.