views.py

import botocore
import datetime
import json
import logging
import os
import uuid

from rest_framework import permissions, status
from rest_framework.decorators import (
    api_view,
    authentication_classes,
    permission_classes,
    throttle_classes,
)

from django.conf import settings
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import transaction, IntegrityError
from django.db.models import Count
from django.utils import timezone

from rest_framework_expiring_authtoken.authentication import (
    ExpiringTokenAuthentication,
)
from rest_framework_simplejwt.authentication import JWTAuthentication
from rest_framework.response import Response
from rest_framework.throttling import UserRateThrottle, AnonRateThrottle
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema

from accounts.permissions import HasVerifiedEmail
from base.utils import (
    StandardResultSetPagination,
    get_boto3_client,
    get_or_create_sqs_queue,
    paginated_queryset,
    is_user_a_staff,
)
from challenges.models import (
    ChallengePhase,
    Challenge,
    ChallengeEvaluationCluster,
    ChallengePhaseSplit,
    LeaderboardData,
)
from challenges.utils import (
    complete_s3_multipart_file_upload,
    generate_presigned_url_for_multipart_upload,
    get_aws_credentials_for_challenge,
    get_challenge_model,
    get_challenge_phase_model,
    get_challenge_phase_split_model,
    get_participant_model,
)
from hosts.models import ChallengeHost
from hosts.utils import is_user_a_host_of_challenge, is_user_a_staff_or_host
from participants.models import ParticipantTeam
from participants.utils import (
    get_participant_team_model,
    get_participant_team_id_of_user_for_a_challenge,
    get_participant_team_of_user_for_a_challenge,
    is_user_part_of_participant_team,
)
from .aws_utils import generate_aws_eks_bearer_token
from .filters import SubmissionFilter
from .models import Submission
from .sender import publish_submission_message
from .serializers import (
    CreateLeaderboardDataSerializer,
    LeaderboardDataSerializer,
    RemainingSubmissionDataSerializer,
    SubmissionSerializer,
)
from .tasks import download_file_and_publish_submission_message
from .utils import (
    calculate_distinct_sorted_leaderboard_data,
    get_leaderboard_data_model,
    get_remaining_submission_for_a_phase,
    get_submission_model,
    handle_submission_rerun,
    handle_submission_resume,
    is_url_valid,
    reorder_submissions_comparator,
    reorder_submissions_comparator_to_key,
)

logger = logging.getLogger(__name__)


@swagger_auto_schema(
    methods=["post"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_id",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        ),
        openapi.Parameter(
            name="challenge_phase_id",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge Phase ID",
            required=True,
        ),
    ],
    responses={status.HTTP_201_CREATED: openapi.Response("")},
)
@swagger_auto_schema(
    methods=["get"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_id",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        ),
        openapi.Parameter(
            name="challenge_phase_id",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge Phase ID",
            required=True,
        ),
    ],
    responses={status.HTTP_201_CREATED: openapi.Response("")},
)
@api_view(["GET", "POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def challenge_submission(request, challenge_id, challenge_phase_id):
    """API Endpoint for making a submission to a challenge"""

    # check if the challenge exists or not
    try:
        challenge = Challenge.objects.get(pk=challenge_id)
    except Challenge.DoesNotExist:
        response_data = {"error": "Challenge does not exist"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    # check if the challenge phase exists or not
    try:
        challenge_phase = ChallengePhase.objects.get(
            pk=challenge_phase_id, challenge=challenge
        )
    except ChallengePhase.DoesNotExist:
        response_data = {"error": "Challenge Phase does not exist"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if request.method == "GET":
        # getting participant team object for the user for a particular challenge.
        participant_team_id = get_participant_team_id_of_user_for_a_challenge(
            request.user, challenge_id
        )

        # check if participant team exists or not.
        try:
            ParticipantTeam.objects.get(pk=participant_team_id)
        except ParticipantTeam.DoesNotExist:
            response_data = {
                "error": "You haven't participated in the challenge"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        submission = Submission.objects.filter(
            participant_team=participant_team_id,
            challenge_phase=challenge_phase,
            ignore_submission=False,
        ).order_by("-submitted_at")
        filtered_submissions = SubmissionFilter(
            request.GET, queryset=submission
        )
        # rerank in progress submissions in ascending order of submitted_at
        reordered_submissions = sorted(
            filtered_submissions.qs,
            key=reorder_submissions_comparator_to_key(
                reorder_submissions_comparator
            ),
        )
        paginator, result_page = paginated_queryset(
            reordered_submissions, request
        )
        serializer = SubmissionSerializer(
            result_page, many=True, context={"request": request}
        )
        response_data = serializer.data
        return paginator.get_paginated_response(response_data)

    elif request.method == "POST":

        # check if the challenge is active or not
        if not challenge.is_active:
            response_data = {"error": "Challenge is not active"}
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        # check if challenge phase is active
        if not challenge_phase.is_active:
            response_data = {
                "error": "Sorry, cannot accept submissions since challenge phase is not active"
            }
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        # check if user is a challenge host or a participant
        if not is_user_a_host_of_challenge(request.user, challenge_id):
            # check if challenge phase is public and accepting solutions
            if not challenge_phase.is_public:
                response_data = {
                    "error": "Sorry, cannot accept submissions since challenge phase is not public"
                }
                return Response(
                    response_data, status=status.HTTP_403_FORBIDDEN
                )

            # if allowed email ids list exist, check if the user exist in that list or not
            if challenge_phase.allowed_email_ids:
                if request.user.email not in challenge_phase.allowed_email_ids:
                    response_data = {
                        "error": "Sorry, you are not allowed to participate in this challenge phase"
                    }
                    return Response(
                        response_data, status=status.HTTP_403_FORBIDDEN
                    )

        participant_team_id = get_participant_team_id_of_user_for_a_challenge(
            request.user, challenge_id
        )
        try:
            participant_team = ParticipantTeam.objects.get(
                pk=participant_team_id
            )
        except ParticipantTeam.DoesNotExist:
            response_data = {
                "error": "You haven't participated in the challenge"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        # check if manual approval is enabled and team is approved
        if challenge.manual_participant_approval and not challenge.approved_participant_teams.filter(pk=participant_team_id).exists():
            response_data = {
                "error": "Your team is not approved by challenge host"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        all_participants_email = participant_team.get_all_participants_email()
        for participant_email in all_participants_email:
            if participant_email in challenge.banned_email_ids:
                message = "You're a part of {} team and it has been banned from this challenge. \
                Please contact the challenge host.".format(
                    participant_team.team_name
                )
                response_data = {"error": message}
                return Response(
                    response_data, status=status.HTTP_403_FORBIDDEN
                )

        # Fetch the number of submissions under progress.
        submissions_in_progress_status = [
            Submission.SUBMITTED,
            Submission.SUBMITTING,
            Submission.RESUMING,
            Submission.QUEUED,
            Submission.RUNNING,
        ]
        submissions_in_progress = Submission.objects.filter(
            participant_team=participant_team_id,
            challenge_phase=challenge_phase,
            status__in=submissions_in_progress_status,
        ).count()

        if (
            submissions_in_progress
            >= challenge_phase.max_concurrent_submissions_allowed
        ):
            message = "You have {} submissions that are being processed. \
                       Please wait for them to finish and then try again."
            response_data = {"error": message.format(submissions_in_progress)}
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        if not request.FILES:
            if request.data.get("file_url") is None:
                response_data = {"error": "The file URL is missing!"}
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )
            if not is_url_valid(request.data["file_url"]):
                response_data = {"error": "The file URL does not exists!"}
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )
            download_file_and_publish_submission_message.delay(
                request.data,
                request.user.id,
                request.method,
                challenge_phase_id,
            )
            response_data = {
                "message": "Please wait while your submission being evaluated!"
            }
            return Response(response_data, status=status.HTTP_200_OK)

        if request.data.get("submission_meta_attributes"):
            submission_meta_attributes = json.load(
                request.data.get("submission_meta_attributes")
            )
            request.data[
                "submission_meta_attributes"
            ] = submission_meta_attributes

        if request.data.get("is_public") is None:
            request.data["is_public"] = (
                True if challenge_phase.is_submission_public else False
            )
        else:
            request.data["is_public"] = json.loads(request.data["is_public"])
            if (
                request.data.get("is_public")
                and challenge_phase.is_restricted_to_select_one_submission
            ):
                # Handle corner case for restrict one public lb submission
                submissions_already_public = Submission.objects.filter(
                    is_public=True,
                    participant_team=participant_team,
                    challenge_phase=challenge_phase,
                )
                # Make the existing public submission private before making the new submission public
                if submissions_already_public.count() == 1:
                    # Case when the phase is restricted to make only one submission as public
                    submission_serializer = SubmissionSerializer(
                        submissions_already_public[0],
                        data={"is_public": False},
                        context={
                            "participant_team": participant_team,
                            "challenge_phase": challenge_phase,
                            "request": request,
                        },
                        partial=True,
                    )
                    if submission_serializer.is_valid():
                        submission_serializer.save()

        # Override submission visibility if leaderboard_public = False for a challenge phase
        if not challenge_phase.leaderboard_public:
            request.data["is_public"] = challenge_phase.is_submission_public

        serializer = SubmissionSerializer(
            data=request.data,
            context={
                "participant_team": participant_team,
                "challenge_phase": challenge_phase,
                "request": request,
            },
        )
        message = {
            "challenge_pk": challenge_id,
            "phase_pk": challenge_phase_id,
            "is_static_dataset_code_upload_submission": False,
        }
        if challenge.is_docker_based:
            try:
                file_content = json.loads(request.FILES["input_file"].read())
                message["submitted_image_uri"] = file_content[
                    "submitted_image_uri"
                ]
                if challenge.is_static_dataset_code_upload:
                    message["is_static_dataset_code_upload_submission"] = True
            except Exception as ex:
                response_data = {
                    "error": "Error {} in submitted_image_uri from submission file".format(
                        ex
                    )
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

        if serializer.is_valid():
            serializer.save()
            response_data = serializer.data
            submission = serializer.instance
            message["submission_pk"] = submission.id
            # publish message in the queue
            publish_submission_message(message)
            return Response(response_data, status=status.HTTP_201_CREATED)
        return Response(
            serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE
        )


@api_view(["PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def change_submission_data_and_visibility(
    request, challenge_pk, challenge_phase_pk, submission_pk
):
    """
    API Endpoint for updating the submission meta data
    and changing submission visibility.
    """

    # check if the challenge exists or not
    challenge = get_challenge_model(challenge_pk)

    # check if the challenge phase exists or not
    challenge_phase = get_challenge_phase_model(challenge_phase_pk)

    if not challenge.is_active:
        response_data = {"error": "Challenge is not active"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    # check if challenge phase is public and accepting solutions
    if not is_user_a_host_of_challenge(request.user, challenge_pk):
        if not challenge_phase.is_public:
            response_data = {
                "error": "Sorry, cannot accept submissions since challenge phase is not public"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)
        elif request.data.get("is_baseline"):
            response_data = {
                "error": "Sorry, you are not authorized to make this request"
            }
            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    participant_team_pk = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge_pk
    )

    try:
        participant_team = ParticipantTeam.objects.get(pk=participant_team_pk)
    except ParticipantTeam.DoesNotExist:
        response_data = {"error": "You haven't participated in the challenge"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    try:
        submission = Submission.objects.get(
            participant_team=participant_team,
            challenge_phase=challenge_phase,
            id=submission_pk,
        )
    except Submission.DoesNotExist:
        response_data = {"error": "Submission does not exist"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    try:
        is_public = request.data["is_public"]
        if is_public is True:
            when_made_public = datetime.datetime.now()
            request.data["when_made_public"] = when_made_public

            submissions_already_public = Submission.objects.filter(
                is_public=True,
                participant_team=participant_team,
                challenge_phase=challenge_phase,
            )
            # Make the existing public submission private before making the new submission public
            if (
                challenge_phase.is_restricted_to_select_one_submission
                and is_public
                and submissions_already_public.count() == 1
            ):
                # Case when the phase is restricted to make only one submission as public
                submission_serializer = SubmissionSerializer(
                    submissions_already_public[0],
                    data={"is_public": False},
                    context={
                        "participant_team": participant_team,
                        "challenge_phase": challenge_phase,
                        "request": request,
                    },
                    partial=True,
                )
                if submission_serializer.is_valid():
                    submission_serializer.save()
    except KeyError:
        pass

    serializer = SubmissionSerializer(
        submission,
        data=request.data,
        context={
            "participant_team": participant_team,
            "challenge_phase": challenge_phase,
            "request": request,
        },
        partial=True,
    )

    if serializer.is_valid():
        serializer.save()
        response_data = serializer.data
        return Response(response_data, status=status.HTTP_200_OK)
    else:
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


@swagger_auto_schema(
    methods=["get"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_phase_split_id",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge Phase Split ID",
            required=True,
        )
    ],
    operation_id="leaderboard",
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="",
            schema=openapi.Schema(
                type=openapi.TYPE_OBJECT,
                properties={
                    "count": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="Count of values on the leaderboard",
                    ),
                    "next": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="URL of next page of results",
                    ),
                    "previous": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="URL of previous page of results",
                    ),
                    "results": openapi.Schema(
                        type=openapi.TYPE_ARRAY,
                        description="Array of results object",
                        items=openapi.Schema(
                            type=openapi.TYPE_OBJECT,
                            properties={
                                "submission__participant_team__team_name": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Participant Team Name",
                                ),
                                "challenge_phase_split": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Challenge Phase Split ID",
                                ),
                                "filtering_score": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Default filtering score for results",
                                ),
                                "leaderboard__schema": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Leaderboard Schema of the corresponding challenge",
                                ),
                                "result": openapi.Schema(
                                    type=openapi.TYPE_ARRAY,
                                    description="Leaderboard Metrics values according to leaderboard schema",
                                    items=openapi.Schema(
                                        type=openapi.TYPE_OBJECT
                                    ),
                                ),
                                "submission__submitted_at": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Time stamp when submission was submitted at",
                                ),
                            },
                        ),
                    ),
                },
            ),
        )
    },
)
@api_view(["GET"])
@throttle_classes([AnonRateThrottle, UserRateThrottle])
def leaderboard(request, challenge_phase_split_id):
    """
    Returns leaderboard for a corresponding Challenge Phase Split

    - Arguments:
        ``challenge_phase_split_id``: Primary key for the challenge phase split for which leaderboard is to be fetched

    - Returns:
        Leaderboard entry objects in a list
    """

    # check if the challenge exists or not
    challenge_phase_split = get_challenge_phase_split_model(
        challenge_phase_split_id
    )
    challenge_obj = challenge_phase_split.challenge_phase.challenge
    order_by = request.GET.get("order_by")
    (
        response_data,
        http_status_code,
    ) = calculate_distinct_sorted_leaderboard_data(
        request.user,
        challenge_obj,
        challenge_phase_split,
        only_public_entries=True,
        order_by=order_by,
    )
    # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard.
    if http_status_code == status.HTTP_400_BAD_REQUEST:
        return Response(response_data, status=http_status_code)

    paginator, result_page = paginated_queryset(
        response_data, request, pagination_class=StandardResultSetPagination()
    )
    response_data = result_page
    return paginator.get_paginated_response(response_data)


@swagger_auto_schema(
    methods=["get"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_phase_split_pk",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge Phase Split Primary Key",
            required=True,
        )
    ],
    operation_id="get_all_entries_on_public_leaderboard",
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="",
            schema=openapi.Schema(
                type=openapi.TYPE_OBJECT,
                properties={
                    "count": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="Count of values on the leaderboard",
                    ),
                    "next": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="URL of next page of results",
                    ),
                    "previous": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="URL of previous page of results",
                    ),
                    "results": openapi.Schema(
                        type=openapi.TYPE_ARRAY,
                        description="Array of results object",
                        items=openapi.Schema(
                            type=openapi.TYPE_OBJECT,
                            properties={
                                "id": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Result ID",
                                ),
                                "submission__participant_team": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Participant Team ID",
                                ),
                                "submission__participant_team__team_name": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Participant Team Name",
                                ),
                                "submission__participant_team__team_url": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Participant Team URL",
                                ),
                                "submission__is_baseline": openapi.Schema(
                                    type=openapi.TYPE_BOOLEAN,
                                    description="Boolean to decide if submission is baseline",
                                ),
                                "submission__is_public": openapi.Schema(
                                    type=openapi.TYPE_BOOLEAN,
                                    description="Boolean to decide if submission is public",
                                ),
                                "challenge_phase_split": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Challenge Phase Split ID",
                                ),
                                "result": openapi.Schema(
                                    type=openapi.TYPE_ARRAY,
                                    description="Leaderboard Metrics values according to leaderboard schema",
                                    items=openapi.Schema(
                                        type=openapi.TYPE_STRING
                                    ),
                                ),
                                "error": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Error returned for the result",
                                ),
                                "leaderboard__schema": openapi.Schema(
                                    type=openapi.TYPE_OBJECT,
                                    description="Leaderboard Schema of the corresponding challenge",
                                    properties={
                                        "labels": openapi.Schema(
                                            type=openapi.TYPE_ARRAY,
                                            description="Labels of leaderboard schema",
                                            items=openapi.Schema(
                                                type=openapi.TYPE_STRING
                                            ),
                                        ),
                                        "default_order_by": openapi.Schema(
                                            type=openapi.TYPE_STRING,
                                            description="Default ordering label for the leaderboard schema",
                                        ),
                                    },
                                ),
                                "submission__submitted_at": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Time stamp when submission was submitted at",
                                ),
                                "submission__method_name": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Method of submission",
                                ),
                                "submission__id": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="ID of submission",
                                ),
                                "submission__submission_metadata": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Metadata and other info about submission",
                                ),
                                "filtering_score": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Default filtering score for results",
                                ),
                                "filtering_error": openapi.Schema(
                                    type=openapi.TYPE_STRING,
                                    description="Default filtering error for results",
                                ),
                            },
                        ),
                    ),
                },
            ),
        ),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@api_view(["GET"])
@throttle_classes([AnonRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_all_entries_on_public_leaderboard(request, challenge_phase_split_pk):
    """
    Returns public/private leaderboard entries to corresponding challenge phase split for a challenge host

    - Arguments:
        ``challenge_phase_split_pk``: Primary key for the challenge phase split for which leaderboard is to be fetched

    - Returns:
        All Leaderboard entry objects in a list

    Below is the sample response returned by the API

    ```
    {
    "count": 1,
    "next": null,
    "previous": null,
    "results": [
        {
            "id": 1,
            "submission__participant_team": 2,
            "submission__participant_team__team_name": "Sanchezview Participant Team",
            "submission__participant_team__team_url": "",
            "submission__is_baseline": true,
            "submission__is_public": true,
            "challenge_phase_split": 1,
            "result": [
                26
            ],
            "error": null,
            "leaderboard__schema": {
                "labels": [
                    "score"
                ],
                "default_order_by": "score"
            },
            "submission__submitted_at": "2021-01-12T10:14:58.764572Z",
            "submission__method_name": "Vernon",
            "submission__id": 10,
            "submission__submission_metadata": null,
            "filtering_score": 26.0,
            "filtering_error": 0
        }
    ]
    }
    ```
    """
    # check if the challenge exists or not
    challenge_phase_split = get_challenge_phase_split_model(
        challenge_phase_split_pk
    )

    challenge_obj = challenge_phase_split.challenge_phase.challenge

    # Allow access only to challenge host
    if not is_user_a_staff_or_host(request.user, challenge_obj.pk):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
    order_by = request.GET.get("order_by")
    (
        response_data,
        http_status_code,
    ) = calculate_distinct_sorted_leaderboard_data(
        request.user,
        challenge_obj,
        challenge_phase_split,
        only_public_entries=False,
        order_by=order_by,
    )
    # The response 400 will be returned if the leaderboard isn't public or `default_order_by` key is missing in leaderboard.
    if http_status_code == status.HTTP_400_BAD_REQUEST:
        return Response(response_data, status=http_status_code)

    paginator, result_page = paginated_queryset(
        response_data, request, pagination_class=StandardResultSetPagination()
    )
    response_data = result_page
    return paginator.get_paginated_response(response_data)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_remaining_submissions(request, challenge_pk):
    """
    API to get the number of remaining submission for all phases.
    Below is the sample response returned by the API

    {
        "participant_team": "Sample_Participant_Team",
        "participant_team_id": 2,
        "phases": [
            {
                "id": 1,
                "slug": "megan-phase-1",
                "name": "Megan Phase",
                "start_date": "2018-10-28T14:22:53.022639Z",
                "end_date": "2020-06-19T14:22:53.022660Z",
                "limits": {
                    "remaining_submissions_this_month_count": 9,
                    "remaining_submissions_today_count": 5,
                    "remaining_submissions_count": 29
                }
            },
            {
                "id": 2,
                "slug": "molly-phase-2",
                "name": "Molly Phase",
                "start_date": "2018-10-28T14:22:53Z",
                "end_date": "2020-06-19T14:22:53Z",
                "limits": {
                    "message": "You have exhausted this month's submission limit!",
                    "remaining_time": "1481076.929224"  // remaining_time is in seconds
                }
            }
        ]
    }
    """
    phases_data = {}
    challenge = get_challenge_model(challenge_pk)
    challenge_phases = ChallengePhase.objects.filter(
        challenge=challenge
    ).order_by("pk")
    if not is_user_a_host_of_challenge(request.user, challenge_pk):
        challenge_phases = challenge_phases.filter(
            challenge=challenge, is_public=True
        ).order_by("pk")
    phase_data_list = list()
    for phase in challenge_phases:
        (
            remaining_submission_message,
            response_status,
        ) = get_remaining_submission_for_a_phase(
            request.user, phase.id, challenge_pk
        )
        if response_status != status.HTTP_200_OK:
            return Response(
                remaining_submission_message, status=response_status
            )
        phase_data_list.append(
            RemainingSubmissionDataSerializer(
                phase, context={"limits": remaining_submission_message}
            ).data
        )
    phases_data["phases"] = phase_data_list
    participant_team = get_participant_team_of_user_for_a_challenge(
        request.user, challenge_pk
    )
    phases_data["participant_team"] = participant_team.team_name
    phases_data["participant_team_id"] = participant_team.id
    return Response(phases_data, status=status.HTTP_200_OK)


@api_view(["GET", "DELETE"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_submission_by_pk(request, submission_id):
    """
    API endpoint to fetch the details of a submission.
    Only the submission owner or the challenge hosts are allowed.
    """
    try:
        submission = Submission.objects.get(pk=submission_id)
    except Submission.DoesNotExist:
        response_data = {
            "error": "Submission {} does not exist".format(submission_id)
        }
        return Response(response_data, status=status.HTTP_404_NOT_FOUND)

    host_team = submission.challenge_phase.challenge.creator
    if (
        request.user.id == submission.created_by.id
        or ChallengeHost.objects.filter(
            user=request.user.id, team_name__pk=host_team.pk
        ).exists()
    ):
        if request.method == "GET":
            serializer = SubmissionSerializer(
                submission, context={"request": request}
            )
            response_data = serializer.data
            return Response(response_data, status=status.HTTP_200_OK)

        elif request.method == "DELETE":
            serializer = SubmissionSerializer(
                submission,
                data=request.data,
                context={"ignore_submission": True, "request": request},
                partial=True,
            )
            if serializer.is_valid():
                serializer.save()
                response_data = serializer.data
                return Response(response_data, status=status.HTTP_200_OK)
            return Response(
                serializer.errors, status=status.HTTP_400_BAD_REQUEST
            )
    response_data = {
        "error": "Sorry, you are not authorized to access this submission."
    }
    return Response(response_data, status=status.HTTP_401_UNAUTHORIZED)


@swagger_auto_schema(
    methods=["put"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_pk",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        )
    ],
    operation_id="update_submission",
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        properties={
            "challenge_phase": openapi.Schema(
                type=openapi.TYPE_STRING, description="Challenge Phase ID"
            ),
            "submission": openapi.Schema(
                type=openapi.TYPE_STRING, description="Submission ID"
            ),
            "stdout": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Submission output file content",
            ),
            "stderr": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Submission error file content",
            ),
            "submission_status": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Final status of submission (can take one of these values): CANCELLED/FAILED/FINISHED",
            ),
            "result": openapi.Schema(
                type=openapi.TYPE_ARRAY,
                description="Submission results in array format."
                " API will throw an error if any split and/or metric is missing)",
                items=openapi.Schema(
                    type=openapi.TYPE_OBJECT,
                    properties={
                        "split1": openapi.Schema(
                            type=openapi.TYPE_STRING,
                            description="dataset split 1 codename",
                        ),
                        "show_to_participant": openapi.Schema(
                            type=openapi.TYPE_BOOLEAN,
                            description="Boolean to decide if the results are shown to participant or not",
                        ),
                        "accuracies": openapi.Schema(
                            type=openapi.TYPE_OBJECT,
                            description="Accuracies on different metrics",
                            properties={
                                "metric1": openapi.Schema(
                                    type=openapi.TYPE_NUMBER,
                                    description="Numeric accuracy on metric 1",
                                ),
                                "metric2": openapi.Schema(
                                    type=openapi.TYPE_NUMBER,
                                    description="Numeric accuracy on metric 2",
                                ),
                            },
                        ),
                    },
                ),
            ),
            "metadata": openapi.Schema(
                type=openapi.TYPE_OBJECT,
                description="It contains the metadata related to submission (only visible to challenge hosts)",
                properties={
                    "foo": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="Some data relevant to key",
                    )
                },
            ),
        },
    ),
    responses={
        status.HTTP_200_OK: openapi.Response(
            "{'success': 'Submission result has been successfully updated'}"
        ),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@swagger_auto_schema(
    methods=["patch"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_pk",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        )
    ],
    operation_id="update_submission",
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        properties={
            "submission": openapi.Schema(
                type=openapi.TYPE_STRING, description="Submission ID"
            ),
            "job_name": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Job name for the running submission",
            ),
            "submission_status": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Updated status of submission from submitted i.e. RUNNING",
            ),
        },
    ),
    responses={
        status.HTTP_200_OK: openapi.Response("{<updated submission-data>}"),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@api_view(["PUT", "PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def update_submission(request, challenge_pk):
    """
    API endpoint to update submission related attributes

    Query Parameters:

     - ``challenge_phase``: challenge phase id, e.g. 123 (**required**)
     - ``submission``: submission id, e.g. 123 (**required**)
     - ``stdout``: Stdout after evaluation, e.g. "Evaluation completed in 2 minutes" (**required**)
     - ``stderr``: Stderr after evaluation, e.g. "Failed due to incorrect file format" (**required**)
     - ``environment_log``: Environment error after evaluation, e.g. "Failed due to attempted action being invalid" (**code upload challenge only**)
     - ``submission_status``: Status of submission after evaluation
        (can take one of the following values: `FINISHED`/`CANCELLED`/`FAILED`), e.g. FINISHED (**required**)
     - ``result``: contains accuracies for each metric, (**required**) e.g.
            [
                {
                    "split": "split1-codename",
                    "show_to_participant": True,
                    "accuracies": {
                        "metric1": 90
                    }
                },
                {
                    "split": "split2-codename",
                    "show_to_participant": False,
                    "accuracies": {
                        "metric1": 50,
                        "metric2": 40
                    }
                }
            ]
     - ``metadata``: Contains the metadata related to submission (only visible to challenge hosts) e.g:
            {
                "average-evaluation-time": "5 sec",
                "foo": "bar"
            }
    """
    if not is_user_a_staff(request.user) and not is_user_a_host_of_challenge(request.user, challenge_pk):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if request.method == "PUT":
        challenge_phase_pk = request.data.get("challenge_phase")
        submission_pk = request.data.get("submission")
        submission_status = request.data.get("submission_status", "").lower()
        stdout_content = request.data.get("stdout", "").encode("utf-8")
        stderr_content = request.data.get("stderr", "").encode("utf-8")
        environment_log_content = request.data.get("environment_log", "").encode("utf-8")
        submission_result = request.data.get("result", "")
        metadata = request.data.get("metadata", "")
        submission = get_submission_model(submission_pk)

        public_results = []
        successful_submission = (
            True if submission_status == Submission.FINISHED else False
        )
        if submission_status not in [
            Submission.FAILED,
            Submission.CANCELLED,
            Submission.FINISHED,
        ]:
            response_data = {"error": "Sorry, submission status is invalid"}
            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

        if successful_submission:
            try:
                results = json.loads(submission_result)
            except (ValueError, TypeError) as exc:
                response_data = {
                    "error": "`result` key contains invalid data with error {}."
                    "Please try again with correct format.".format(str(exc))
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

            leaderboard_data_list = []
            for phase_result in results:
                split = phase_result.get("split")
                accuracies = phase_result.get("accuracies")
                show_to_participant = phase_result.get(
                    "show_to_participant", False
                )
                try:
                    challenge_phase_split = ChallengePhaseSplit.objects.get(
                        challenge_phase__pk=challenge_phase_pk,
                        dataset_split__codename=split,
                    )
                except ChallengePhaseSplit.DoesNotExist:
                    response_data = {
                        "error": "Challenge Phase Split does not exist with phase_id: {} and"
                        "split codename: {}".format(challenge_phase_pk, split)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                leaderboard_metrics = (
                    challenge_phase_split.leaderboard.schema.get("labels")
                )
                missing_metrics = []
                malformed_metrics = []
                for metric, value in accuracies.items():
                    if metric not in leaderboard_metrics:
                        missing_metrics.append(metric)

                    if not (
                        isinstance(value, float) or isinstance(value, int)
                    ):
                        malformed_metrics.append((metric, type(value)))

                if len(missing_metrics):
                    response_data = {
                        "error": "Following metrics are missing in the"
                        "leaderboard data: {}".format(missing_metrics)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                if len(malformed_metrics):
                    response_data = {
                        "error": "Values for following metrics are not of"
                        "float/int: {}".format(malformed_metrics)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                try:
                    leaderboard_data = get_leaderboard_data_model(
                        submission_pk, challenge_phase_split.pk
                    )
                except LeaderboardData.DoesNotExist:
                    leaderboard_data = None

                data = {"result": accuracies}
                if leaderboard_data is not None:
                    serializer = CreateLeaderboardDataSerializer(
                        leaderboard_data,
                        data=data,
                        partial=True,
                        context={
                            "challenge_phase_split": challenge_phase_split,
                            "submission": submission,
                            "request": request,
                        },
                    )
                else:
                    serializer = CreateLeaderboardDataSerializer(
                        data=data,
                        context={
                            "challenge_phase_split": challenge_phase_split,
                            "submission": submission,
                            "request": request,
                        },
                    )
                if serializer.is_valid():
                    leaderboard_data_list.append(serializer)
                else:
                    return Response(
                        serializer.errors, status=status.HTTP_400_BAD_REQUEST
                    )

                # Only after checking if the serializer is valid, append the public split results to results file
                if show_to_participant:
                    public_results.append(accuracies)

            try:
                with transaction.atomic():
                    for serializer in leaderboard_data_list:
                        serializer.save()
            except IntegrityError:
                logger.exception(
                    "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                )
                response_data = {
                    "error": "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

        submission.status = submission_status
        submission.completed_at = timezone.now()
        submission.stdout_file.save("stdout.txt", ContentFile(stdout_content))
        submission.stderr_file.save("stderr.txt", ContentFile(stderr_content))
        submission.environment_log_file.save("environment_log.txt", ContentFile(environment_log_content))
        submission.submission_result_file.save(
            "submission_result.json", ContentFile(str(public_results))
        )
        submission.submission_metadata_file.save(
            "submission_metadata_file.json", ContentFile(str(metadata))
        )
        submission.save()
        response_data = {
            "success": "Submission result has been successfully updated"
        }
        return Response(response_data, status=status.HTTP_200_OK)

    if request.method == "PATCH":
        submission_pk = request.data.get("submission")
        submission = get_submission_model(submission_pk)
        # Update submission_input_file for is_static_dataset_code_upload submission evaluation
        if (
            request.FILES.get("submission_input_file")
            and submission.challenge_phase.challenge.is_static_dataset_code_upload
        ):
            serializer = SubmissionSerializer(
                submission,
                data=request.data,
                context={
                    "request": request,
                },
                partial=True,
            )
            if serializer.is_valid():
                serializer.save()
                message = {
                    "challenge_pk": challenge_pk,
                    "phase_pk": submission.challenge_phase.pk,
                    "submission_pk": submission_pk,
                    "is_static_dataset_code_upload_submission": False,
                }
                # publish message in the queue
                publish_submission_message(message)
                response_data = serializer.data
                return Response(response_data, status=status.HTTP_200_OK)
            else:
                return Response(
                    serializer.errors, status=status.HTTP_400_BAD_REQUEST
                )
        submission_status = request.data.get("submission_status", "").lower()
        job_name = request.data.get("job_name", "").lower()
        jobs = submission.job_name
        if job_name:
            jobs.append(job_name)
        if submission_status not in [Submission.QUEUED, Submission.RUNNING]:
            response_data = {"error": "Sorry, submission status is invalid"}
            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

        data = {
            "status": submission_status,
            "started_at": str(timezone.now()),
            "job_name": jobs,
        }
        serializer = SubmissionSerializer(
            submission, data=data, partial=True, context={"request": request}
        )
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


@swagger_auto_schema(
    methods=["put"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_pk",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        )
    ],
    operation_id="update_submission",
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        properties={
            "challenge_phase": openapi.Schema(
                type=openapi.TYPE_STRING, description="Challenge Phase ID"
            ),
            "submission": openapi.Schema(
                type=openapi.TYPE_STRING, description="Submission ID"
            ),
            "stdout": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Submission output file content",
            ),
            "stderr": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Submission error file content",
            ),
            "submission_status": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Final status of submission (can take one of these values): CANCELLED/FAILED/FINISHED",
            ),
            "result": openapi.Schema(
                type=openapi.TYPE_ARRAY,
                description="Submission results in array format."
                " API will throw an error if any split and/or metric is missing)",
                items=openapi.Schema(
                    type=openapi.TYPE_OBJECT,
                    properties={
                        "split1": openapi.Schema(
                            type=openapi.TYPE_STRING,
                            description="dataset split 1 codename",
                        ),
                        "show_to_participant": openapi.Schema(
                            type=openapi.TYPE_BOOLEAN,
                            description="Boolean to decide if the results are shown to participant or not",
                        ),
                        "accuracies": openapi.Schema(
                            type=openapi.TYPE_OBJECT,
                            description="Accuracies on different metrics",
                            properties={
                                "metric1": openapi.Schema(
                                    type=openapi.TYPE_NUMBER,
                                    description="Numeric accuracy on metric 1",
                                ),
                                "metric2": openapi.Schema(
                                    type=openapi.TYPE_NUMBER,
                                    description="Numeric accuracy on metric 2",
                                ),
                            },
                        ),
                    },
                ),
            ),
            "metadata": openapi.Schema(
                type=openapi.TYPE_OBJECT,
                description="It contains the metadata related to submission (only visible to challenge hosts)",
                properties={
                    "foo": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="Some data relevant to key",
                    )
                },
            ),
        },
    ),
    responses={
        status.HTTP_200_OK: openapi.Response(
            "{'success': 'Submission result has been successfully updated'}"
        ),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@swagger_auto_schema(
    methods=["patch"],
    manual_parameters=[
        openapi.Parameter(
            name="challenge_pk",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Challenge ID",
            required=True,
        )
    ],
    operation_id="update_submission",
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        properties={
            "submission": openapi.Schema(
                type=openapi.TYPE_STRING, description="Submission ID"
            ),
            "job_name": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Job name for the running submission",
            ),
            "submission_status": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Updated status of submission from submitted i.e. RUNNING",
            ),
        },
    ),
    responses={
        status.HTTP_200_OK: openapi.Response("{<updated submission-data>}"),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@api_view(["PUT", "PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def update_partially_evaluated_submission(request, challenge_pk):
    """
    API endpoint to update submission related attributes

    Query Parameters:

     - ``challenge_phase``: challenge phase id, e.g. 123 (**required**)
     - ``submission``: submission id, e.g. 123 (**required**)
     - ``stdout``: Stdout after evaluation, e.g. "Evaluation completed in 2 minutes" (**required**)
     - ``stderr``: Stderr after evaluation, e.g. "Failed due to incorrect file format" (**required**)
     - ``submission_status``: Status of submission after evaluation
        (can take one of the following values: `FINISHED`/`CANCELLED`/`FAILED`/`PARTIALLY_EVALUATED`),
        e.g. FINISHED (**required**)
     - ``result``: contains accuracies for each metric, (**required**) e.g.
            [
                {
                    "split": "split1-codename",
                    "show_to_participant": True,
                    "accuracies": {
                        "metric1": 90
                    }
                },
                {
                    "split": "split2-codename",
                    "show_to_participant": False,
                    "accuracies": {
                        "metric1": 50,
                        "metric2": 40
                    }
                }
            ]
     - ``metadata``: Contains the metadata related to submission (only visible to challenge hosts) e.g:
            {
                "average-evaluation-time": "5 sec",
                "foo": "bar"
            }
    """
    if not is_user_a_host_of_challenge(request.user, challenge_pk):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if request.method == "PUT":
        challenge_phase_pk = request.data.get("challenge_phase")
        submission_pk = request.data.get("submission")
        submission_status = request.data.get("submission_status", "").lower()
        stdout_content = request.data.get("stdout", "")
        stderr_content = request.data.get("stderr", "")
        submission_result = request.data.get("result", "")
        metadata = request.data.get("metadata", "")
        submission = get_submission_model(submission_pk)

        public_results = []
        successful_submission = (
            True
            if (
                submission_status == Submission.FINISHED
                or submission_status == Submission.PARTIALLY_EVALUATED
            )
            else False
        )
        if submission_status not in [
            Submission.FAILED,
            Submission.CANCELLED,
            Submission.FINISHED,
            Submission.PARTIALLY_EVALUATED,
        ]:
            response_data = {"error": "Sorry, submission status is invalid"}
            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

        if successful_submission:
            try:
                results = json.loads(submission_result)
            except (ValueError, TypeError) as exc:
                response_data = {
                    "error": "`result` key contains invalid data with error {}."
                    "Please try again with correct format.".format(str(exc))
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

            leaderboard_data_list = []
            for phase_result in results:
                split = phase_result.get("split")
                accuracies = phase_result.get("accuracies")
                show_to_participant = phase_result.get(
                    "show_to_participant", False
                )
                try:
                    challenge_phase_split = ChallengePhaseSplit.objects.get(
                        challenge_phase__pk=challenge_phase_pk,
                        dataset_split__codename=split,
                    )
                except ChallengePhaseSplit.DoesNotExist:
                    response_data = {
                        "error": "Challenge Phase Split does not exist with phase_id: {} and"
                        "split codename: {}".format(challenge_phase_pk, split)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                leaderboard_metrics = (
                    challenge_phase_split.leaderboard.schema.get("labels")
                )
                missing_metrics = []
                malformed_metrics = []
                for metric, value in accuracies.items():
                    if metric not in leaderboard_metrics:
                        missing_metrics.append(metric)

                    if not (
                        isinstance(value, float) or isinstance(value, int)
                    ):
                        malformed_metrics.append((metric, type(value)))

                is_partial_evaluation_phase = (
                    challenge_phase_split.challenge_phase.is_partial_submission_evaluation_enabled
                )
                if len(missing_metrics) and not is_partial_evaluation_phase:
                    response_data = {
                        "error": "Following metrics are missing in the"
                        "leaderboard data: {} of challenge phase: {}".format(
                            missing_metrics, challenge_phase_pk
                        )
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                if len(malformed_metrics):
                    response_data = {
                        "error": "Values for following metrics are not of"
                        "float/int: {}".format(malformed_metrics)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                try:
                    leaderboard_data = get_leaderboard_data_model(
                        submission_pk, challenge_phase_split.pk
                    )
                except LeaderboardData.DoesNotExist:
                    leaderboard_data = None

                data = {"result": accuracies}
                if leaderboard_data is not None:
                    serializer = CreateLeaderboardDataSerializer(
                        leaderboard_data,
                        data=data,
                        partial=True,
                        context={
                            "challenge_phase_split": challenge_phase_split,
                            "submission": submission,
                            "request": request,
                        },
                    )
                else:
                    serializer = CreateLeaderboardDataSerializer(
                        data=data,
                        context={
                            "challenge_phase_split": challenge_phase_split,
                            "submission": submission,
                            "request": request,
                        },
                    )
                if serializer.is_valid():
                    leaderboard_data_list.append(serializer)
                else:
                    return Response(
                        serializer.errors, status=status.HTTP_400_BAD_REQUEST
                    )

                # Only after checking if the serializer is valid, append the public split results to results file
                if show_to_participant:
                    public_results.append(accuracies)

            try:
                with transaction.atomic():
                    for serializer in leaderboard_data_list:
                        serializer.save()
            except IntegrityError:
                logger.exception(
                    "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                )
                response_data = {
                    "error": "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

        submission.status = submission_status
        submission.completed_at = timezone.now()
        submission.stdout_file.save("stdout.txt", ContentFile(stdout_content))
        submission.stderr_file.save("stderr.txt", ContentFile(stderr_content))
        submission.submission_result_file.save(
            "submission_result.json", ContentFile(str(public_results))
        )
        submission.submission_metadata_file.save(
            "submission_metadata_file.json", ContentFile(str(metadata))
        )
        submission.save()
        response_data = {
            "success": "Submission result has been successfully updated"
        }
        return Response(response_data, status=status.HTTP_200_OK)

    if request.method == "PATCH":
        submission_pk = request.data.get("submission")
        submission_status = request.data.get("submission_status", "").lower()
        job_name = request.data.get("job_name", "").lower()
        submission = get_submission_model(submission_pk)
        jobs = submission.job_name
        if job_name:
            jobs.append(job_name)
        if submission_status not in [
            Submission.RUNNING,
            Submission.PARTIALLY_EVALUATED,
            Submission.FINISHED,
        ]:
            response_data = {"error": "Sorry, submission status is invalid"}
            return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

        if submission_status == Submission.RUNNING:
            data = {
                "status": submission_status,
                "started_at": str(timezone.now()),
                "job_name": jobs,
            }
            serializer = SubmissionSerializer(
                submission,
                data=data,
                partial=True,
                context={"request": request},
            )
            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data, status=status.HTTP_200_OK)
            else:
                return Response(
                    serializer.errors, status=status.HTTP_400_BAD_REQUEST
                )
        elif (
            submission_status == Submission.PARTIALLY_EVALUATED
            or submission_status == Submission.FINISHED
        ):
            challenge_phase_pk = request.data.get("challenge_phase")
            stdout_content = request.data.get("stdout", "")
            stderr_content = request.data.get("stderr", "")
            submission_result = request.data.get("result", "")

            try:
                results = json.loads(submission_result)
            except (ValueError, TypeError) as exc:
                response_data = {
                    "error": "`result` key contains invalid data with error {}."
                    "Please try again with correct format.".format(str(exc))
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

            public_results = []
            leaderboard_data_list = []
            for phase_result in results:
                split = phase_result.get("split")
                accuracies = phase_result.get("accuracies")
                show_to_participant = phase_result.get(
                    "show_to_participant", False
                )
                try:
                    challenge_phase_split = ChallengePhaseSplit.objects.get(
                        challenge_phase__pk=challenge_phase_pk,
                        dataset_split__codename=split,
                    )
                except ChallengePhaseSplit.DoesNotExist:
                    response_data = {
                        "error": "Challenge Phase Split does not exist with phase_id: {} and"
                        "split codename: {}".format(challenge_phase_pk, split)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                try:
                    leaderboard_data = get_leaderboard_data_model(
                        submission_pk, challenge_phase_split.pk
                    )
                except LeaderboardData.DoesNotExist:
                    response_data = {
                        "error": "Leaderboard Data does not exist with phase_id: {} and"
                        "submission id: {}".format(
                            challenge_phase_pk, submission_pk
                        )
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                updated_result = leaderboard_data.result
                leaderboard_metrics = (
                    challenge_phase_split.leaderboard.schema.get("labels")
                )
                missing_metrics = []
                malformed_metrics = []
                for metric, value in accuracies.items():
                    if metric not in leaderboard_metrics:
                        missing_metrics.append(metric)

                    if not (
                        isinstance(value, float) or isinstance(value, int)
                    ):
                        malformed_metrics.append((metric, type(value)))
                    updated_result[metric] = value

                is_partial_evaluation_phase = (
                    challenge_phase_split.challenge_phase.is_partial_submission_evaluation_enabled
                )
                if len(missing_metrics) and not is_partial_evaluation_phase:
                    response_data = {
                        "error": "Following metrics are missing in the"
                        "leaderboard data: {} of challenge phase: {}".format(
                            missing_metrics, challenge_phase_pk
                        )
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                if len(malformed_metrics):
                    response_data = {
                        "error": "Values for following metrics are not of"
                        "float/int: {}".format(malformed_metrics)
                    }
                    return Response(
                        response_data, status=status.HTTP_400_BAD_REQUEST
                    )

                data = {"result": updated_result}
                serializer = CreateLeaderboardDataSerializer(
                    leaderboard_data,
                    data=data,
                    partial=True,
                    context={
                        "challenge_phase_split": challenge_phase_split,
                        "submission": submission,
                        "request": request,
                    },
                )
                if serializer.is_valid():
                    leaderboard_data_list.append(serializer)
                else:
                    return Response(
                        serializer.errors, status=status.HTTP_400_BAD_REQUEST
                    )

                # Only after checking if the serializer is valid, append the public split results to results file
                if show_to_participant:
                    public_results.append(accuracies)

            try:
                with transaction.atomic():
                    for serializer in leaderboard_data_list:
                        serializer.save()
            except IntegrityError:
                logger.exception(
                    "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                )
                response_data = {
                    "error": "Failed to update submission_id {} related metadata".format(
                        submission_pk
                    )
                }
                return Response(
                    response_data, status=status.HTTP_400_BAD_REQUEST
                )

            submission.status = submission_status
            submission.completed_at = timezone.now()
            submission.stdout_file.save(
                "stdout.txt", ContentFile(stdout_content)
            )
            submission.stderr_file.save(
                "stderr.txt", ContentFile(stderr_content)
            )
            submission.submission_result_file.save(
                "submission_result.json", ContentFile(str(public_results))
            )
            submission.save()
            response_data = {
                "success": "Submission result has been successfully updated"
            }
            return Response(response_data, status=status.HTTP_200_OK)
        response_data = {"error": "Sorry, submission status is invalid"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)


@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def re_run_submission(request, submission_pk):
    """
    API endpoint to re-run a submission.
    Only challenge host has access to this endpoint by default.
    Participants can submit if the challenge allows.
    """
    try:
        submission = Submission.objects.get(pk=submission_pk)
    except Submission.DoesNotExist:
        response_data = {
            "error": "Submission {} does not exist".format(submission_pk)
        }
        return Response(response_data, status=status.HTTP_404_NOT_FOUND)

    if submission.ignore_submission:
        response_data = {
            "error": "Deleted submissions can't be re-run"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    # get the challenge and challenge phase object
    challenge_phase = submission.challenge_phase
    challenge = challenge_phase.challenge

    if not challenge.allow_participants_resubmissions and not is_user_a_staff_or_host(request.user, challenge.pk):
        response_data = {
            "error": "Only challenge hosts or admins are allowed to re-run a submission"
        }
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    if not challenge.is_active:
        response_data = {
            "error": "Challenge {} is not active".format(challenge.title)
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    message = handle_submission_rerun(submission, Submission.CANCELLED)
    publish_submission_message(message)
    response_data = {
        "success": "Submission is successfully submitted for re-running"
    }
    return Response(response_data, status=status.HTTP_200_OK)


@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def resume_submission(request, submission_pk):
    """
    API endpoint to resume a submission from failed or partially evaluated state.
    Only challenge host has access to this endpoint.
    """
    try:
        submission = Submission.objects.get(pk=submission_pk)
    except Submission.DoesNotExist:
        response_data = {
            "error": "Submission {} does not exist".format(submission_pk)
        }
        return Response(response_data, status=status.HTTP_404_NOT_FOUND)

    if submission.ignore_submission:
        response_data = {
            "error": "Deleted submissions can't be resumed"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if submission.status != Submission.FAILED:
        response_data = {
            "error": "Only failed submissions can be resumed"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if submission.status == Submission.RESUMING:
        response_data = {
            "error": "Submission is already resumed"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    # get the challenge and challenge phase object
    challenge_phase = submission.challenge_phase
    challenge = challenge_phase.challenge

    if not challenge.allow_participants_resubmissions and not is_user_a_host_of_challenge(request.user, challenge.pk):
        response_data = {
            "error": "Only challenge hosts are allowed to resume a submission"
        }
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    if not challenge.is_active:
        response_data = {
            "error": "Challenge {} is not active".format(challenge.title)
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    if not challenge.remote_evaluation:
        response_data = {
            "error": "Challenge {} is not remote. Resuming is only supported for remote challenges.".format(challenge.title)
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    if not challenge.allow_resuming_submissions:
        response_data = {
            "error": "Challenge {} does not allow resuming submissions.".format(challenge.title)
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    message = handle_submission_resume(submission, Submission.RESUMING)
    publish_submission_message(message)
    response_data = {
        "success": "Submission is successfully resumed"
    }
    return Response(response_data, status=status.HTTP_200_OK)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_submissions_for_challenge(request, challenge_pk):

    challenge = get_challenge_model(challenge_pk)

    if not is_user_a_staff(request.user) and not is_user_a_host_of_challenge(request.user, challenge.id):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    submission_status = request.query_params.get("status", None)

    valid_submission_status = [
        Submission.SUBMITTED,
        Submission.RUNNING,
        Submission.QUEUED,
        Submission.RESUMING,
        Submission.FAILED,
        Submission.CANCELLED,
        Submission.FINISHED,
        Submission.SUBMITTING,
    ]

    if submission_status not in valid_submission_status:
        response_data = {
            "error": "Invalid submission status {}".format(submission_status)
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    submissions_done_in_challenge = Submission.objects.filter(
        challenge_phase__challenge=challenge.id, status=submission_status
    )

    serializer = SubmissionSerializer(
        submissions_done_in_challenge, many=True, context={"request": request}
    )

    return Response(serializer.data, status=status.HTTP_200_OK)


@swagger_auto_schema(
    methods=["get"],
    manual_parameters=[
        openapi.Parameter(
            name="queue_name",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Queue Name",
            required=True,
        )
    ],
    operation_id="get_submission_message_from_queue",
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="",
            schema=openapi.Schema(
                type=openapi.TYPE_OBJECT,
                properties={
                    "body": openapi.Schema(
                        type=openapi.TYPE_OBJECT,
                        description="SQS message queue dict object",
                        properties={
                            "challenge_pk": openapi.Schema(
                                type=openapi.TYPE_INTEGER,
                                description="Primary key for the challenge in the database",
                            ),
                            "phase_pk": openapi.Schema(
                                type=openapi.TYPE_INTEGER,
                                description="Primary key for the challenge phase in the database",
                            ),
                            "submission_pk": openapi.Schema(
                                type=openapi.TYPE_INTEGER,
                                description="Primary key for the submission in the database",
                            ),
                            "submitted_image_uri": openapi.Schema(
                                type=openapi.TYPE_STRING,
                                description="The AWS ECR URL for the pushed docker image in docker based challenges",
                            ),
                        },
                    ),
                    "receipt_handle": openapi.Schema(
                        type=openapi.TYPE_STRING,
                        description="SQS message receipt handle",
                    ),
                },
            ),
        ),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_submission_message_from_queue(request, queue_name):
    """
    API to fetch submission message from AWS SQS queue.

    - Arguments:
        ``queue_name``: AWS SQS queue name

    - Returns:
        ``body``: The message body content as a key-value pair
        ``receipt_handle``: The message receipt handle
    """
    try:
        challenge = Challenge.objects.get(queue=queue_name)  # noqa
    except Challenge.DoesNotExist:
        response_data = {
            "error": "Challenge with queue name {} does not exist".format(
                queue_name
            )
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        response_data = {
            "error": "Sorry, you are not authorized to access this resource"
        }
        return Response(response_data, status=status.HTTP_401_UNAUTHORIZED)

    queue = get_or_create_sqs_queue(queue_name, challenge)
    try:
        messages = queue.receive_messages()
        if len(messages):
            message_receipt_handle = messages[0].receipt_handle
            message_body = json.loads(messages[0].body)
            logger.info(
                "A submission is received with pk {}".format(
                    message_body.get("submission_pk")
                )
            )
        else:
            logger.info("No submission received")
            message_receipt_handle = None
            message_body = None

        response_data = {
            "body": message_body,
            "receipt_handle": message_receipt_handle,
        }
        return Response(response_data, status=status.HTTP_200_OK)
    except botocore.exceptions.ClientError as ex:
        response_data = {"error": ex}
        logger.exception("Exception raised: {}".format(ex))
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)


@swagger_auto_schema(
    methods=["post"],
    manual_parameters=[
        openapi.Parameter(
            name="queue_name",
            in_=openapi.IN_PATH,
            type=openapi.TYPE_STRING,
            description="Queue Name",
            required=True,
        )
    ],
    operation_id="delete_submission_message_from_queue",
    request_body=openapi.Schema(
        type=openapi.TYPE_OBJECT,
        properties={
            "receipt_handle": openapi.Schema(
                type=openapi.TYPE_STRING,
                description="Receipt handle for the message to be deleted",
            )
        },
    ),
    responses={
        status.HTTP_200_OK: openapi.Response(
            "{'success': 'Message deleted successfully from the queue <queue-name>'}"
        ),
        status.HTTP_400_BAD_REQUEST: openapi.Response(
            "{'error': 'Error message goes here'}"
        ),
    },
)
@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def delete_submission_message_from_queue(request, queue_name):
    """
    API to delete submission message from AWS SQS queue

    - Arguments:
        ``queue_name``  -- The unique authentication token provided by challenge hosts

    - Request Body:
        ``receipt_handle`` -- The receipt handle of the message to be deleted
    """
    try:
        challenge = Challenge.objects.get(queue=queue_name)
    except Challenge.DoesNotExist:
        response_data = {
            "error": "Challenge with queue name {} does not exists".format(
                queue_name
            )
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    challenge_pk = challenge.pk
    receipt_handle = request.data["receipt_handle"]
    if not receipt_handle:
        response_data = {
            "error": "Please add message receipt handle in the body"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if not is_user_a_host_of_challenge(request.user, challenge_pk):
        response_data = {
            "error": "Sorry, you are not authorized to access this resource"
        }
        return Response(response_data, status=status.HTTP_401_UNAUTHORIZED)

    queue = get_or_create_sqs_queue(queue_name, challenge)
    try:
        message = queue.Message(receipt_handle)
        message.delete()
        response_data = {
            "success": "Message deleted successfully from the queue: {}".format(
                queue_name
            )
        }
        return Response(response_data, status=status.HTTP_200_OK)
    except botocore.exceptions.ClientError as ex:
        response_data = {"error": ex}
        logger.exception(
            "SQS message is not deleted due to {}".format(response_data)
        )
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_signed_url_for_submission_related_file(request):
    """Returns S3 signed URL for a particular file residing on S3 bucket

    Arguments:
        request {object} -- Request object

    Returns:
        Response object -- Response object with appropriate response code (200/400/403/404)
    """

    # Assumption: file will be stored in this format: 'team_{id}/submission_{id}/.../file.log'
    bucket = request.query_params.get("bucket", None)
    key = request.query_params.get("key", None)

    if not bucket or not key:
        response_data = {"error": "key and bucket names can't be empty"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    try:
        splits = key.split("/")
        participant_team_id, submission_id = (
            splits[0].replace("team_", ""),
            splits[1].replace("submission_", ""),
        )
    except Exception:
        response_data = {
            "error": "Invalid file path format. Please try again with correct file path format."
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    participant_team = get_participant_team_model(participant_team_id)
    submission = get_submission_model(submission_id)
    challenge_pk = submission.challenge_phase.challenge.pk

    if submission.participant_team != participant_team:
        response_data = {
            "error": "You are not authorized to access this file."
        }
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    if is_user_part_of_participant_team(
        request.user, participant_team
    ) or is_user_a_host_of_challenge(request.user, challenge_pk):
        aws_keys = get_aws_credentials_for_challenge(challenge_pk)
        s3 = get_boto3_client("s3", aws_keys)
        url = s3.generate_presigned_url(
            ClientMethod="get_object", Params={"Bucket": bucket, "Key": key}
        )
        response_data = {"signed_url": url}
        return Response(response_data, status=status.HTTP_200_OK)
    else:
        response_data = {
            "error": "You are not authorized to access this file."
        }
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)


@api_view(["PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def update_leaderboard_data(request, leaderboard_data_pk):
    """API endpoint to update a metric in leaderboard data

    Arguments:
        request {HttpRequest} -- The request object
        leaderboard_data_pk {int} -- Primary key from leaderboard data table
    """

    try:
        leaderboard_data = LeaderboardData.objects.get(pk=leaderboard_data_pk, is_disabled=False)
    except LeaderboardData.DoesNotExist:
        response_data = {"error": "Leaderboard data does not exist"}
        return Response(response_data, status=status.HTTP_404_NOT_FOUND)

    challenge = (
        leaderboard_data.challenge_phase_split.challenge_phase.challenge
    )

    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    data = request.data.get("leaderboard_data")
    if data is None:
        response_data = {"error": "leaderboard_data can't be blank"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    try:
        data = json.loads(data)
    except (ValueError, TypeError) as exc:
        response_data = {
            "error": "`leaderboard_data` key contains invalid data with error {}."
            "Please try again with correct format.".format(str(exc))
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
    leaderboard_metrics = leaderboard_data.leaderboard.schema.get("labels")
    missing_metrics = []
    extra_metrics = []
    malformed_metrics = []
    for metric in leaderboard_metrics:
        if metric not in data:
            missing_metrics.append(metric)

    for metric, value in data.items():
        if metric not in leaderboard_metrics:
            extra_metrics.append(metric)

        if not (isinstance(value, float) or isinstance(value, int)):
            malformed_metrics.append((metric, type(value)))

    if len(missing_metrics) and len(extra_metrics):
        response_data = {
            "error": "Following metrics {0} are missing and following metrics are invalid {1} in the "
            "leaderboard data".format(missing_metrics, extra_metrics)
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if len(missing_metrics):
        response_data = {
            "error": "Following metrics are missing in the "
            "leaderboard data: {}".format(missing_metrics)
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if len(extra_metrics):
        response_data = {
            "error": "Following metrics are invalid in the "
            "leaderboard data: {}".format(extra_metrics)
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if len(malformed_metrics):
        response_data = {
            "error": "Values for following metrics are not of"
            "float/int: {}".format(malformed_metrics)
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)
    result = {"result": data}
    serializer = LeaderboardDataSerializer(
        leaderboard_data,
        data=result,
        partial=True,
        context={"request": request},
    )
    if serializer.is_valid():
        serializer.save()
        return Response(serializer.data, status=status.HTTP_200_OK)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_bearer_token(request, challenge_pk):
    """API to generate and return bearer token AWS EKS requests

    Arguments:
        request {HttpRequest} -- The request object
        challenge_pk {int} -- The challenge pk for which bearer token is to be generated

    Returns:
        Response object -- Response object with appropriate response code (200/400/404)
    """
    challenge = get_challenge_model(challenge_pk)

    if not is_user_a_host_of_challenge(request.user, challenge.id):
        response_data = {
            "error": "Sorry, you are not authorized to make this request!"
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if not challenge.is_docker_based:
        response_data = {
            "error": "The challenge doesn't require uploading Docker images, hence there isn't a need for bearer token."
        }
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    try:
        challenge_evaluation_cluster = ChallengeEvaluationCluster.objects.get(
            challenge=challenge
        )
    except ChallengeEvaluationCluster.DoesNotExist:
        response_data = {
            "error": "Challenge evaluation cluster for the challenge with pk {} does not exist".format(
                challenge.pk
            )
        }
        return Response(response_data, status=status.HTTP_404_NOT_FOUND)

    cluster_name = challenge_evaluation_cluster.name
    bearer_token = generate_aws_eks_bearer_token(cluster_name, challenge)
    response_data = {
        "aws_eks_bearer_token": bearer_token,
        "cluster_name": cluster_name,
    }

    return Response(response_data, status=status.HTTP_200_OK)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_github_badge_data(
    request, challenge_phase_split_pk, participant_team_pk
):
    """
    Add API to get data for dynamically generating github badges
    Ref: https://shields.io/endpoint
    Arguments:
        request {HttpRequest} -- The request object
        phase_pk {[int]} -- Challenge phase primary key
        participant_team_pk {[int]} -- Participant team primary key

    Returns:
        {dict} -- A dict which contains keys schemaVersion, label, message and color of badge
    """
    challenge_phase_split = get_challenge_phase_split_model(
        challenge_phase_split_pk
    )
    challenge_obj = challenge_phase_split.challenge_phase.challenge
    data = {"schemaVersion": 1, "label": "EvalAI", "color": "blue"}

    (
        response_data,
        http_status_code,
    ) = calculate_distinct_sorted_leaderboard_data(
        request.user,
        challenge_obj,
        challenge_phase_split,
        only_public_entries=True,
        order_by=None,
    )
    if http_status_code == status.HTTP_400_BAD_REQUEST:
        return Response(response_data, status=http_status_code)

    for idx, team_data in enumerate(response_data):
        if team_data["submission__participant_team"] == int(
            participant_team_pk
        ):
            data["message"] = "{} Rank #{}".format(
                challenge_obj.title, idx + 1
            )
            break
        else:
            data["message"] = challenge_obj.title
    return Response(data, status=http_status_code)


@api_view(["GET"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def challenge_phase_submission_count_by_status(request, challenge_phase_pk):
    """
    API for fetching count of submissions by status for a challenge phase

    Arguments:
        request {HttpRequest} -- request object
        challenge_phase_pk {int} -- challenge phase pk

    Returns:
        Response object -- Response object with appropriate response code (200/400/404)
    """
    # check if the challenge phase exists or not
    challenge_phase = get_challenge_phase_model(challenge_phase_pk)

    challenge = challenge_phase.challenge

    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        response_data = {
            "error": "Sorry, you are not authorized to make this request"
        }
        return Response(response_data, status=status.HTTP_401_UNAUTHORIZED)

    submissions = (
        Submission.objects.filter(challenge_phase=challenge_phase)
        .values("status")
        .annotate(count=Count("id"))
    )

    response_data = {"status": submissions}
    return Response(response_data, status=status.HTTP_200_OK)


@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def get_submission_file_presigned_url(request, challenge_phase_pk):
    """
    API to generate a presigned url to upload a submission file

    Arguments:
        request {HttpRequest} -- The request object
        challenge_phase_pk {int} -- Challenge phase primary key
    Returns:
         Response Object -- An object containing the presignd url and submission id, or an error message if some failure occurs
    """
    if settings.DEBUG:
        response_data = {
            "error": "Sorry, this feature is not available in development or test environment."
        }
        return Response(response_data)

    # Check if the challenge phase exists or not
    challenge_phase = get_challenge_phase_model(challenge_phase_pk)

    challenge = challenge_phase.challenge

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )

    if not challenge.is_active:
        response_data = {"error": "Challenge is not active"}
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    # Check if challenge phase is active
    if not challenge_phase.is_active:
        response_data = {
            "error": "Sorry, cannot accept submissions since challenge phase is not active"
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    # Check if user is a challenge host or a participant
    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        # Check if challenge phase is public and accepting solutions
        if not challenge_phase.is_public:
            response_data = {
                "error": "Sorry, cannot accept submissions since challenge phase is not public"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        if not challenge.approved_by_admin:
            response_data = {
                "error": "Challenge is not yet approved by admin."
            }
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        # if allowed email ids list exist, check if the user exist in that list or not
        if challenge_phase.allowed_email_ids:
            if request.user.email not in challenge_phase.allowed_email_ids:
                response_data = {
                    "error": "Sorry, you are not allowed to participate in this challenge phase"
                }
                return Response(
                    response_data, status=status.HTTP_403_FORBIDDEN
                )

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )
    try:
        participant_team = ParticipantTeam.objects.get(pk=participant_team_id)
    except ParticipantTeam.DoesNotExist:
        response_data = {"error": "You haven't participated in the challenge"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    all_participants_email = participant_team.get_all_participants_email()
    for participant_email in all_participants_email:
        if participant_email in challenge.banned_email_ids:
            message = "You're a part of {} team and it has been banned from this challenge. \
            Please contact the challenge host.".format(
                participant_team.team_name
            )
            response_data = {"error": message}
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    # Fetch the number of submissions under progress.
    submissions_in_progress_status = [
        Submission.SUBMITTED,
        Submission.SUBMITTING,
        Submission.RESUMING,
        Submission.QUEUED,
        Submission.RUNNING,
    ]
    submissions_in_progress = Submission.objects.filter(
        participant_team=participant_team_id,
        challenge_phase=challenge_phase,
        status__in=submissions_in_progress_status,
    ).count()

    if (
        submissions_in_progress
        >= challenge_phase.max_concurrent_submissions_allowed
    ):
        message = "You have {} submissions that are being processed. \
                   Please wait for them to finish and then try again."
        response_data = {"error": message.format(submissions_in_progress)}
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    file_ext = os.path.splitext(request.data["file_name"])[-1]
    random_file_name = uuid.uuid4()
    # This file shall be replaced with the one uploaded through the presigned url from the CLI
    input_file = SimpleUploadedFile(
        "{}{}".format(random_file_name, file_ext),
        b"file_content",
        content_type="text/plain",
    )
    submission_data = request.data.copy()

    if submission_data.get("is_public") is None:
        submission_data["is_public"] = (
            True if challenge_phase.is_submission_public else False
        )
    else:
        submission_data["is_public"] = json.loads(request.data["is_public"])

    # Override submission visibility if leaderboard_public = False for a challenge phase
    if not challenge_phase.leaderboard_public:
        submission_data["is_public"] = challenge_phase.is_submission_public

    submission_data["input_file"] = input_file
    serializer = SubmissionSerializer(
        data=submission_data,
        context={
            "participant_team": participant_team,
            "challenge_phase": challenge_phase,
            "request": request,
        },
    )
    # Set default num of chunks to 1 if num of chunks is not specified
    num_file_chunks = 1
    if request.data.get("num_file_chunks"):
        num_file_chunks = int(request.data["num_file_chunks"])

    response = {}
    if serializer.is_valid():
        serializer.save()
        submission = serializer.instance

        file_key_on_s3 = "{}/{}".format(
            settings.MEDIAFILES_LOCATION, submission.input_file.name
        )
        response = generate_presigned_url_for_multipart_upload(
            file_key_on_s3, challenge.pk, num_file_chunks
        )
        if response.get("error"):
            response_data = response
            response = Response(
                response_data, status=status.HTTP_400_BAD_REQUEST
            )
        else:
            response_data = {
                "presigned_urls": response.get("presigned_urls"),
                "upload_id": response.get("upload_id"),
                "submission_pk": submission.pk,
            }
            response = Response(response_data, status=status.HTTP_201_CREATED)
        return response
    response_data = {"error": serializer.errors}
    return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)


@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def finish_submission_file_upload(request, challenge_phase_pk, submission_pk):
    """
    API to complete multipart upload of presigned url submission

    Arguments:
        request {HttpRequest} -- The request object
        challenge_phase_pk {int} -- Challenge phase primary key
        submission_pk {int} -- Submission primary key
    Returns:
         Response Object -- An object containing the presignd url and submission id, or an error message if some failure occurs
    """
    if settings.DEBUG:
        response_data = {
            "error": "Sorry, this feature is not available in development or test environment."
        }
        return Response(response_data)

    # Check if the challenge phase exists or not
    challenge_phase = get_challenge_phase_model(challenge_phase_pk)

    challenge = challenge_phase.challenge

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )

    if not challenge.is_active:
        response_data = {"error": "Challenge is not active"}
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    # Check if challenge phase is active
    if not challenge_phase.is_active:
        response_data = {
            "error": "Sorry, cannot accept submissions since challenge phase is not active"
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    # Check if user is a challenge host or a participant
    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        # Check if challenge phase is public and accepting solutions
        if not challenge_phase.is_public:
            response_data = {
                "error": "Sorry, cannot accept submissions since challenge phase is not public"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        if not challenge.approved_by_admin:
            response_data = {
                "error": "Challenge is not yet approved by admin."
            }
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        # if allowed email ids list exist, check if the user exist in that list or not
        if challenge_phase.allowed_email_ids:
            if request.user.email not in challenge_phase.allowed_email_ids:
                response_data = {
                    "error": "Sorry, you are not allowed to participate in this challenge phase"
                }
                return Response(
                    response_data, status=status.HTTP_403_FORBIDDEN
                )

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )
    try:
        participant_team = ParticipantTeam.objects.get(pk=participant_team_id)
    except ParticipantTeam.DoesNotExist:
        response_data = {"error": "You haven't participated in the challenge"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    all_participants_email = participant_team.get_all_participants_email()
    for participant_email in all_participants_email:
        if participant_email in challenge.banned_email_ids:
            message = "You're a part of {} team and it has been banned from this challenge. \
            Please contact the challenge host.".format(
                participant_team.team_name
            )
            response_data = {"error": message}
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    if request.data.get("parts") is None:
        response_data = {"error": "Uploaded file Parts metadata is missing"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    if request.data.get("upload_id") is None:
        response_data = {"error": "Uploaded file UploadId is missing"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    file_parts = json.loads(request.data["parts"])
    upload_id = request.data["upload_id"]
    response = {}
    try:
        submission = get_submission_model(submission_pk)
        file_key_on_s3 = "{}/{}".format(
            settings.MEDIAFILES_LOCATION, submission.input_file.name
        )
        data = complete_s3_multipart_file_upload(
            file_parts, upload_id, file_key_on_s3, challenge.pk
        )
        if data.get("error"):
            response_data = data
            response = Response(
                response_data, status=status.HTTP_400_BAD_REQUEST
            )
        else:
            response_data = {
                "upload_id": upload_id,
                "submission_pk": submission.pk,
            }
            response = Response(response_data, status=status.HTTP_201_CREATED)
    except Submission.DoesNotExist:
        response_data = {"error": "Submission does not exist"}
        response = Response(response_data, status=status.HTTP_400_BAD_REQUEST)
    return response


@api_view(["POST"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def send_submission_message(request, challenge_phase_pk, submission_pk):
    """
    API to send a submisison message to the challenge specific SQS queue

    Arguments:
        request {HttpRequest} -- The request object
        challenge_phase_pk {int} -- Challenge phase primary key
        submission_pk {int} -- Submission primary key
    Returns:
         Response Object -- An object containing an empty dict and having a HTTP_200_0k status
    """
    try:
        challenge_phase = get_challenge_phase_model(challenge_phase_pk)
    except ChallengePhase.DoesNotExist:
        response_data = {"error": "Challenge Phase does not exist"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    challenge = challenge_phase.challenge

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )

    if not challenge.is_active:
        response_data = {"error": "Challenge is not active"}
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    if not challenge_phase.is_active:
        response_data = {
            "error": "Sorry, cannot accept submissions since challenge phase is not active"
        }
        return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE)

    if not is_user_a_host_of_challenge(request.user, challenge.pk):
        if not challenge_phase.is_public:
            response_data = {
                "error": "Sorry, cannot accept submissions since challenge phase is not public"
            }
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

        if not challenge.approved_by_admin:
            response_data = {
                "error": "Challenge is not yet approved by admin."
            }
            return Response(
                response_data, status=status.HTTP_406_NOT_ACCEPTABLE
            )

        if challenge_phase.allowed_email_ids:
            if request.user.email not in challenge_phase.allowed_email_ids:
                response_data = {
                    "error": "Sorry, you are not allowed to participate in this challenge phase"
                }
                return Response(
                    response_data, status=status.HTTP_403_FORBIDDEN
                )

    participant_team_id = get_participant_team_id_of_user_for_a_challenge(
        request.user, challenge.pk
    )
    try:
        participant_team = ParticipantTeam.objects.get(pk=participant_team_id)
    except ParticipantTeam.DoesNotExist:
        response_data = {"error": "You haven't participated in the challenge"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    all_participants_email = participant_team.get_all_participants_email()
    for participant_email in all_participants_email:
        if participant_email in challenge.banned_email_ids:
            message = "You're a part of {} team and it has been banned from this challenge. \
            Please contact the challenge host.".format(
                participant_team.team_name
            )
            response_data = {"error": message}
            return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    try:
        get_submission_model(submission_pk)
    except Submission.DoesNotExist:
        response_data = {"error": "Submission does not exist"}
        return Response(response_data, status=status.HTTP_400_BAD_REQUEST)

    submission_message = {
        "submission_pk": submission_pk,
        "phase_pk": challenge_phase_pk,
        "challenge_pk": challenge.pk,
    }

    publish_submission_message(submission_message)
    response_data = {}
    return Response(response_data, status=status.HTTP_200_OK)


@api_view(["PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def update_submission_started_at(request, submission_pk):
    """
    API Endpoint for updating the submission evaluation start time.
    """
    try:
        submission = Submission.objects.get(
            id=submission_pk,
        )
    except Submission.DoesNotExist:
        response_data = {"error": "Submission does not exist"}
        return Response(response_data, status=status.HTTP_403_FORBIDDEN)

    serializer = SubmissionSerializer(
        submission,
        data={"started_at": str(timezone.now())},
        context={"request": request},
        partial=True,
    )

    if serializer.is_valid():
        serializer.save()
        response_data = serializer.data
        return Response(response_data, status=status.HTTP_200_OK)
    else:
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


@api_view(["PATCH"])
@throttle_classes([UserRateThrottle])
@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail))
@authentication_classes((JWTAuthentication, ExpiringTokenAuthentication))
def update_submission_meta(request, challenge_pk, submission_pk):
    """
    Common API Endpoint for updating the submission meta data for hosts and participants.
    """

    if is_user_a_host_of_challenge(request.user, challenge_pk):
        submission = get_submission_model(submission_pk)

        serializer = SubmissionSerializer(
            submission,
            data=request.data,
            context={
                "request": request,
            },
            partial=True,
        )

        if serializer.is_valid():
            serializer.save()
            response_data = serializer.data
            return Response(response_data, status=status.HTTP_200_OK)
        else:
            return Response(
                serializer.errors, status=status.HTTP_400_BAD_REQUEST
            )
    else:
        participant_team_pk = get_participant_team_id_of_user_for_a_challenge(
            request.user, challenge_pk
        )

        participant_team = get_participant_model(participant_team_pk)

        try:
            submission = Submission.objects.get(
                id=submission_pk,
                participant_team=participant_team,
            )
        except Submission.DoesNotExist:
            response_data = {
                "error": "Submission {} does not exist".format(submission_pk)
            }
            return Response(response_data, status=status.HTTP_404_NOT_FOUND)

        serializer = SubmissionSerializer(
            submission,
            data=request.data,
            context={"request": request},
            partial=True,
        )

        if serializer.is_valid():
            serializer.save()
            response_data = serializer.data
            return Response(response_data, status=status.HTTP_200_OK)
        else:
            return Response(
                serializer.errors, status=status.HTTP_400_BAD_REQUEST
            )