diff --git a/lambda_function.py b/lambda_function.py index e14ba94..762a32d 100644 --- a/lambda_function.py +++ b/lambda_function.py @@ -27,7 +27,12 @@ def killdeadAlarms(fleetId, monitorapp, project): if eachevent["EventInformation"]["EventSubType"] == "terminated": todel.append(eachevent["EventInformation"]["InstanceId"]) todel = [f"{project}_{x}" for x in todel] - cloudwatch.delete_alarms(AlarmNames=todel) + while len(todel) > 100: + dellist = todel[:100] + cloudwatch.delete_alarms(AlarmNames=dellist) + todel = todel[100:] + if len(todel) <= 100: + cloudwatch.delete_alarms(AlarmNames=todel) print("Old alarms deleted") @@ -115,7 +120,12 @@ def lambda_handler(event, lambda_context): active_instances = [] for instance in active_dictionary["ActiveInstances"]: active_instances.append(instance["InstanceId"]) - cloudwatch.delete_alarms(AlarmNames=active_instances) + while len(active_instances) > 100: + dellist = active_instances[:100] + cloudwatch.delete_alarms(AlarmNames=dellist) + active_instances = active_instances[100:] + if len(active_instances) <= 100: + cloudwatch.delete_alarms(AlarmNames=active_instances) killdeadAlarms(fleetId, monitorapp, project) # Read spot fleet id and terminate all EC2 instances diff --git a/run.py b/run.py index 4da71d4..371a93f 100644 --- a/run.py +++ b/run.py @@ -398,7 +398,7 @@ def create_dashboard(requestInfo): "x": 12, "type": "log", "properties": { - "query": f"SOURCE {APP_NAME} | fields @message| filter @message like 'cellprofiler -c'| stats count_distinct(@message)\n", + "query": f"SOURCE '{APP_NAME}' | fields @message| filter @message like 'cellprofiler -c'| stats count_distinct(@message)\n", "region": AWS_REGION, "stacked": False, "title": "Distinct Logs with \"cellprofiler -c\"", @@ -412,7 +412,7 @@ def create_dashboard(requestInfo): "x": 0, "type": "log", "properties": { - "query": f"SOURCE {APP_NAME} | fields @message| filter @message like 'cellprofiler -c'| stats count(@message)", + "query": f"SOURCE '{APP_NAME}' | fields @message| filter @message like 'cellprofiler -c'| stats count(@message)", "region": AWS_REGION, "stacked": False, "title": "All Logs \"cellprofiler -c\"", @@ -426,7 +426,7 @@ def create_dashboard(requestInfo): "x": 0, "type": "log", "properties": { - "query": f"SOURCE {APP_NAME} | fields @message | filter @message like \"Error\" | display @message", + "query": f"SOURCE '{APP_NAME}' | fields @message | filter @message like \"Error\" | display @message", "region": AWS_REGION, "stacked": False, "title": "Errors", @@ -598,7 +598,7 @@ def startCluster(): createMonitor.write('"MONITOR_QUEUE_NAME" : "'+SQS_QUEUE_NAME+'",\n') createMonitor.write('"MONITOR_BUCKET_NAME" : "'+AWS_BUCKET+'",\n') createMonitor.write('"MONITOR_LOG_GROUP_NAME" : "'+LOG_GROUP_NAME+'",\n') - createMonitor.write('"MONITOR_START_TIME" : "'+ starttime+'"}\n') + createMonitor.write('"MONITOR_START_TIME" : "'+ starttime+'",\n') createMonitor.write('"CLEAN_DASHBOARD" : "'+ CLEAN_DASHBOARD+'"}\n') createMonitor.close() diff --git a/worker/run-worker.sh b/worker/run-worker.sh index 7efa848..9aef692 100644 --- a/worker/run-worker.sh +++ b/worker/run-worker.sh @@ -19,19 +19,13 @@ aws ec2 create-tags --resources $VOL_0_ID --tags Key=Name,Value=${APP_NAME}Worke VOL_1_ID=$(aws ec2 describe-instance-attribute --instance-id $MY_INSTANCE_ID --attribute blockDeviceMapping --output text --query BlockDeviceMappings[1].Ebs.[VolumeId]) aws ec2 create-tags --resources $VOL_1_ID --tags Key=Name,Value=${APP_NAME}Worker -# 2. MOUNT S3 + +# 2. MOUNT S3 +echo $AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY > /credentials.txt +chmod 600 /credentials.txt mkdir -p /home/ubuntu/bucket mkdir -p /home/ubuntu/local_output -if [[ -z "$AWS_ACCESS_KEY_ID" ]] -then - AWS_ACCESS_KEY_ID=$(curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI | jq '.AccessKeyId') - AWS_SECRET_ACCESS_KEY=$(curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI | jq '.SecretAccessKey') - echo $AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY > /credentials.txt - chmod 600 /credentials.txt - stdbuf -o0 s3fs $AWS_BUCKET /home/ubuntu/bucket -o passwd_file=/credentials.txt -o dbglevel=info -else - stdbuf -o0 s3fs $AWS_BUCKET /home/ubuntu/bucket -o ecs -o dbglevel=info -fi +stdbuf -o0 s3fs $AWS_BUCKET /home/ubuntu/bucket -o passwd_file=/credentials.txt # 3. SET UP ALARMS