02.CF-MLOps-Ethnicity-Prediction-Pipeline.yml

AWSTemplateFormatVersion: "2010-09-09"
Description: MLOps-Ethnicity-Prediction-Pipeline
Parameters: 
  UniqueID: 
    Type: String
    Default: yourinitials
    Description: Enter your initials in lower case as a unique identifier for components created 
    AllowedPattern: "^([a-z]|(d(?!d{0,2}.d{1,3}.d{1,3}.d{1,3})))([a-zd]|(.(?!(.|-)))|(-(?!.))){1,61}[a-zd.]$"
  RepositoryBranch:
    Type: String
    Default: master
    Description: Branch to trigger CodePipeline on commit. For this lab, you can leave this as master.
  LambdaFunctionsBucket:
    Type: String
    Default: ethnicity-pred-lambda-code-yourinitials-uniqueid
    Description: S3 Bucket containing your Lambda Functions
  
Resources:
  ElasticContainerRegistry:
    Type: AWS::ECR::Repository
    Properties:
      RepositoryName: 
        Fn::Join:
          - ""
          - - "ethnicity-pred-ecr-"
            - !Ref UniqueID

  CodePipelineArtifactBucket:
    Type: AWS::S3::Bucket
    Properties:
      AccessControl: BucketOwnerFullControl
      VersioningConfiguration: 
        Status: Enabled
      BucketName:
        Fn::Join:
          - ""
          - - "ethnicity-pred-codepipeline-artifacts-"
            - !Ref UniqueID
            - "-"
            - !Select
             - 0
             - !Split
              - "-"
              - !Select
                - 2
                - !Split
                  - "/"
                  - !Ref "AWS::StackId"

  ModelArtifactBucket:
    Type: AWS::S3::Bucket
    Properties:
      AccessControl: BucketOwnerFullControl
      VersioningConfiguration: 
        Status: Enabled
      BucketName:
        Fn::Join:
          - ""
          - - "ethnicity-pred-model-artifacts-"
            - !Ref UniqueID
            - "-"
            - !Select
             - 0
             - !Split
              - "-"
              - !Select
                - 2
                - !Split
                  - "/"
                  - !Ref "AWS::StackId"

  S3DataBucket:
    Type: AWS::S3::Bucket
    Properties: 
      AccessControl: BucketOwnerFullControl
      VersioningConfiguration: 
        Status: Enabled
      BucketName:
        Fn::Join:
          - ""
          - - "ethnicity-pred-data-"
            - !Ref UniqueID
            - "-"
            - !Select
             - 0
             - !Split
              - "-"
              - !Select
                - 2
                - !Split
                  - "/"
                  - !Ref "AWS::StackId"

  CodePipeline:
    Type: AWS::CodePipeline::Pipeline
    Properties: 
      RoleArn: !GetAtt [CodePipelineRole, Arn]
      ArtifactStore:
        Location:
          Ref:
            CodePipelineArtifactBucket
        Type: S3
      Stages: 
        - 
          Name: Source
          Actions: 
            - 
              Name: SourceAction
              ActionTypeId: 
                Category: Source
                Owner: AWS
                Version: 1
                Provider: CodeCommit
              OutputArtifacts: 
                - 
                  Name: SourceArtifact
              Configuration: 
                BranchName: 
                  Ref: RepositoryBranch
                RepositoryName: "ethnicity-pred-codecommit"
              RunOrder: 1
        - 
          Name: Build
          Actions: 
            - 
              Name: BuildImage
              ActionTypeId: 
                Category: Build
                Owner: AWS
                Version: 1
                Provider: CodeBuild
              OutputArtifacts:
                - Name: ImageBuild
              InputArtifacts:
                - Name: SourceArtifact
              Configuration:
                ProjectName: !Ref CodeBuildEPredImage
              RunOrder: 1
        - 
          Name: TrainModel
          Actions: 
            - 
              Name: StartTraining
              ActionTypeId: 
                Category: Invoke
                Owner: AWS
                Version: 1
                Provider: Lambda
              OutputArtifacts:
                - Name: jobEventInfo
              InputArtifacts:
                - Name: ImageBuild
              Configuration:
                FunctionName: 
                  Fn::Join:
                    - ""
                    - - "Ethnicity-Pred-StartTrain-"
                      - !Ref UniqueID
                UserParameters: ' {"traincompute": "ml.c4.xlarge" , "traininstancevolumesize": 10, "traininstancecount": 1} '
              RunOrder: 1
            - 
              Name: GetTrainingStatus
              ActionTypeId: 
                Category: Invoke
                Owner: AWS
                Version: 1
                Provider: Lambda
              OutputArtifacts:
                - Name: CompletedTraining
              InputArtifacts:
                - Name: jobEventInfo
              Configuration:
                FunctionName: 
                  Fn::Join:
                    - ""
                    - - "Ethnicity-Pred-GetStatus-"
                      - !Ref UniqueID
                UserParameters: ' { "stage": "Training"} '
              RunOrder: 2
        - 
          Name: DeployModel-Dev
          Actions: 
            - 
              Name: DeployToDevEnv
              ActionTypeId: 
                Category: Invoke
                Owner: AWS
                Version: 1
                Provider: Lambda
              OutputArtifacts:
                - Name: HostingInfo
              InputArtifacts:
                - Name: CompletedTraining
              Configuration:
                FunctionName: 
                  Fn::Join:
                    - ""
                    - - "Ethnicity-Pred-DeployModel-"
                      - !Ref UniqueID
                UserParameters: '{ "InitialInstanceCount": 1, "InitialVariantWeight": 1, "InstanceType": "ml.t2.medium", "EndpointConfigName": "Dev" }'
              RunOrder: 1
            - 
              Name: GetDeployStatus
              ActionTypeId: 
                Category: Invoke
                Owner: AWS
                Version: 1
                Provider: Lambda
              OutputArtifacts:
                - Name: CompletedDevDeploy
              InputArtifacts:
                - Name: HostingInfo
              Configuration:
                FunctionName: 
                  Fn::Join:
                    - ""
                    - - "Ethnicity-Pred-GetStatus-"
                      - !Ref UniqueID
                UserParameters: ' { "stage": "Deployment"} '
              RunOrder: 2
            - 
              Name: SmokeTest
              ActionTypeId: 
                Category: Invoke
                Owner: AWS
                Version: 1
                Provider: Lambda
              OutputArtifacts:
                - Name: SmokeTestOut
              InputArtifacts:
                - Name: HostingInfo
              Configuration:
                FunctionName:   
                  Fn::Join:
                    - ""
                    - - "Ethnicity-Pred-EvaluateModel-"
                      - !Ref UniqueID
                UserParameters: ' { "env": "Dev" } '
              RunOrder: 3

  CodePipelineRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          Effect: Allow
          Principal:
            Service: codepipeline.amazonaws.com
          Action: sts:AssumeRole
      Path: /
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AdministratorAccess


  CodeBuildEPredImage:
    Type: AWS::CodeBuild::Project
    DependsOn: CodeBuildRole
    Properties:
      Artifacts:
        Type: CODEPIPELINE
      Environment:
        ComputeType: BUILD_GENERAL1_SMALL
        Image: aws/codebuild/python:3.7.1-1.7.0
        Type: LINUX_CONTAINER
        PrivilegedMode: True
        EnvironmentVariables: 
          - Name: AWS_DEFAULT_REGION
            Value: !Ref AWS::Region 
          - Name: IMAGE_REPO_NAME
            Value: !Ref ElasticContainerRegistry 
          - Name: IMAGE_TAG
            Value: "latest" 
          - Name: AWS_ACCOUNT_ID
            Value: !Ref AWS::AccountId
      Source:
        Type: CODEPIPELINE
        BuildSpec: model-code/buildspec.yml
      TimeoutInMinutes: 10
      Name: !Sub ${AWS::StackName}CodeBuildEPredImage
      ServiceRole: !Ref CodeBuildRole

  TrainLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties: 
      Handler: "Ethnicity-Pred-TrainModel.lambda_handler"
      Role: !GetAtt [LambdaExecutionRole, Arn]
      Environment: 
        Variables:
          SageMakerExecutionRole: !GetAtt [SageMakerRole, Arn]
          SSEKMSKeyIdIn: !Ref KMSMasterKeyID
          ECRRepository: !Ref ElasticContainerRegistry
          ModelArtifactBucket: !Ref ModelArtifactBucket
          S3DataBucket: !Ref S3DataBucket
      FunctionName: 
        Fn::Join:
         - ""
         - - "Ethnicity-Pred-StartTrain-"
           - !Ref UniqueID
      Code:
        S3Bucket: !Ref LambdaFunctionsBucket
        S3Key: "Ethnicity-Pred-TrainModel.py.zip"
      Runtime: "python3.9"
      Timeout: 900

  DeployLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties: 
      Handler: "Ethnicity-Pred-DeployModel.lambda_handler"
      Role: !GetAtt [LambdaExecutionRole, Arn]
      Environment: 
        Variables:
          SageMakerExecutionRole: !GetAtt [SageMakerRole, Arn]
          SSEKMSKeyIdIn: !Ref KMSMasterKeyID
      FunctionName: 
        Fn::Join:
         - ""
         - - "Ethnicity-Pred-DeployModel-"
           - !Ref UniqueID
      Code:
        S3Bucket: !Ref LambdaFunctionsBucket
        S3Key: "Ethnicity-Pred-DeployModel.py.zip"
      Runtime: "python3.9"
      Timeout: 900

  GetStatusLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties: 
      Handler: "Ethnicity-Pred-GetStatus.lambda_handler"
      Role: !GetAtt [LambdaExecutionRole, Arn]
      Environment: 
        Variables:
          SageMakerExecutionRole: !GetAtt [SageMakerRole, Arn]
          SSEKMSKeyIdIn: !Ref KMSMasterKeyID
      FunctionName: 
        Fn::Join:
         - ""
         - - "Ethnicity-Pred-GetStatus-"
           - !Ref UniqueID
      Code:
        S3Bucket: !Ref LambdaFunctionsBucket
        S3Key: "Ethnicity-Pred-GetStatus.py.zip"
      Runtime: "python3.9"
      Timeout: 900

  EvaluateModelLambdaFunction:
    Type: "AWS::Lambda::Function"
    Properties: 
      Handler: "Ethnicity-Pred-EvaluateModel.lambda_handler"
      Role: !GetAtt [LambdaExecutionRole, Arn]
      Environment: 
        Variables:
          SageMakerExecutionRole: !GetAtt [SageMakerRole, Arn]
          SSEKMSKeyIdIn: !Ref KMSMasterKeyID
          S3DataBucket: !Ref S3DataBucket
      FunctionName: 
        Fn::Join:
         - ""
         - - "Ethnicity-Pred-EvaluateModel-"
           - !Ref UniqueID
      Code:
        S3Bucket: !Ref LambdaFunctionsBucket 
        S3Key: "Ethnicity-Pred-EvaluateModel.py.zip"
      Runtime: "python3.9"
      Timeout: 900

  CodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          Effect: Allow
          Principal:
            Service: codebuild.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AdministratorAccess
      RoleName: 
        Fn::Join:
          - ""
          - - "EPred-CodeBuild-ServiceRole-"
            - !Ref UniqueID

  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          Effect: Allow
          Principal:
            Service: lambda.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/CloudWatchFullAccess
        - arn:aws:iam::aws:policy/AWSCodePipelineFullAccess
        - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      RoleName: 
       Fn::Join:
          - ""
          - - "EPred-Lambda-ServiceRole-"
            - !Ref UniqueID

  SageMakerRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          Effect: Allow
          Principal:
            Service: sagemaker.amazonaws.com 
          Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/CloudWatchFullAccess
        - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess
      RoleName: 
       Fn::Join:
          - ""
          - - "ethnicity-pred-SageMaker-ServiceRole"
            - !Ref UniqueID

  KMSMasterKeyID:
    Type: AWS::KMS::Key
    Properties: 
      Description: "An example CMK"
      KeyPolicy: 
        Version: "2012-10-17"
        Id: "key-default-1"
        Statement:
          - Sid: "Enable IAM User Permissions"
            Effect: "Allow"
            Principal: 
              AWS: 
                Fn::Join:
                 - ""
                 - - "arn:aws:iam::"
                   - !Ref AWS::AccountId
                   - ":root"
            Action: "kms:*"
            Resource: "*"

Outputs:
      S3DataBucket:
        Description: The ID of the S3 Bucket for model training and test data
        Value: !Ref S3DataBucket
      ElasticContainerRegistry:
        Description: Elastic Container Registry for storing docker images containing training / inference code
        Value: !Ref ElasticContainerRegistry
      ModelArtifactBucket:
        Description: S3 Bucket for storing model ArtifactStore
        Value: !Ref ModelArtifactBucket
      CodePipeline:
        Description: Pipeline created inside the CodePipeline service-role
        Value: !Ref CodePipeline