add commands to manage Git refs (#296)

* add commands to manage Git refs

This changeset adds methods to manage Git refs:
* list of references
* create/update/delete a single reference

* resolve pylint warnings

* resolve flake8 warnings

* remove transcient debugging code

* add unit tests for the refs methods

* adding content to _init_.py file

* delete test/utilities/helper.py as python 2.7.15 doesn't seem to like it

* missing .self

* add ref commands to utCoverage.json

* add vcs test

* fix documentation issues

* address review comments
* renamed reposRefTest.py
* defined more strict assert statements

* fix Not None typo

* allow large response

* fix uppercase in format.py

* Only add column is value exists

* add recording

* adapt to the new way to define organization

* add lock/unlock method and update integration tests

* remove unnecessary stop method

* try to adapt the tests

* Update test_ref.py

* Update commands.py

* Update test_reposRefTest.py

* Update test_ref_createListUpdateDelete.yaml

* Update test_ref.py

Remove commented code

azure-devops/azext_devops/dev/common/git.py

@@ -112,6 +112,27 @@ def get_git_remotes():
     return _git_remotes
 
 
+def resolve_git_refs(ref):
+    """Prepends 'refs/' prefix to ref str if not already there.
+    :param str ref: The text to validate.
+    :rtype: str
+    """
+    if ref is not None and not ref.startswith(REFS_PREFIX):
+        ref = REFS_PREFIX + ref
+    return ref
+
+
+def get_ref_name_from_ref(ref):
+    """Removes 'refs/' prefix from ref str if there.
+    :param ref: The text to validate.
+    :type ref: str
+    :rtype: str
+    """
+    if ref is not None and ref.startswith(REFS_PREFIX):
+        ref = ref[len(REFS_PREFIX):]
+    return ref
+
+
 def resolve_git_ref_heads(ref):
     """Prepends 'refs/heads/' prefix to ref str if not already there.
     :param ref: The text to validate.
@@ -168,6 +189,7 @@ def _get_alias_value(command):
 
 _git_remotes = {}
 _ORIGIN_PUSH_KEY = 'origin(push)'
+REFS_PREFIX = 'refs/'
 REF_HEADS_PREFIX = 'refs/heads/'
 GIT_CREDENTIALS_USERNAME_KEY = 'username'
 GIT_CREDENTIALS_PASSWORD_KEY = 'password'

azure-devops/azext_devops/dev/repos/_format.py

@@ -258,6 +258,39 @@ def _convert_policy_status(status):
     return status.capitalize()
 
 
+def transform_refs_table_output(result):
+    table_output = []
+    for item in sorted(result, key=_get_repo_key):
+        table_output.append(_transform_ref_row(item))
+    return table_output
+
+
+def transform_ref_table_output(result):
+    table_output = [_transform_ref_row(result)]
+    return table_output
+
+
+def _transform_ref_row(row):
+    from azext_devops.dev.common.git import get_ref_name_from_ref
+    table_row = OrderedDict()
+    if 'objectId' in row:
+        table_row['Object ID'] = row['objectId']
+    if ('oldObjectId' in row) and ('newObjectId' in row):
+        old_id = row['oldObjectId']
+        new_id = row['newObjectId']
+        if old_id == '0000000000000000000000000000000000000000':
+            table_row['Object ID'] = new_id
+        elif new_id == '0000000000000000000000000000000000000000':
+            table_row['Object ID'] = old_id
+        else:
+            table_row['Old Object ID'] = old_id
+            table_row['New Object ID'] = new_id
+    table_row['Name'] = get_ref_name_from_ref(row['name'])
+    table_row['Success'] = row['success'] if 'success' in row else None
+    table_row['Update Status'] = row['updateStatus'] if 'updateStatus' in row else None
+    return table_row
+
+
 def transform_repos_table_output(result):
     table_output = []
     for item in sorted(result, key=_get_repo_key):

azure-devops/azext_devops/dev/repos/_help.py

@@ -96,3 +96,9 @@ def load_repos_help():
     short-summary: Manage case enforcement policy.
     long-summary:
     """
+
+    helps['repos ref'] = """
+    type: group
+    short-summary: Manage Git references.
+    long-summary:
+    """

azure-devops/azext_devops/dev/repos/arguments.py

@@ -128,3 +128,7 @@ def load_code_arguments(self, _):
 
     with self.argument_context('repos import create') as context:
         context.argument('git_source_url', options_list=('--git-source-url', '--git-url'))
+
+    with self.argument_context('repos ref') as context:
+        context.argument('repository', options_list=('--repository', '-r'))
+        context.argument('object_id', options_list=('--object-id'))

azure-devops/azext_devops/dev/repos/commands.py

@@ -9,6 +9,8 @@
                       transform_pull_requests_table_output,
                       transform_repo_table_output,
                       transform_repos_table_output,
+                      transform_ref_table_output,
+                      transform_refs_table_output,
                       transform_reviewers_table_output,
                       transform_reviewer_table_output,
                       transform_policies_table_output,
@@ -24,6 +26,10 @@
     exception_handler=azure_devops_exception_handler
 )
 
+reposRefOps = CliCommandType(
+    operations_tmpl='azext_devops.dev.repos.ref#{}'
+)
+
 reposRepositoryOps = CliCommandType(
     operations_tmpl='azext_devops.dev.repos.repository#{}',
     exception_handler=azure_devops_exception_handler
@@ -40,7 +46,7 @@
 )
 
 
-def load_code_commands(self, _):
+def load_code_commands(self, _):  # pylint: disable=too-many-statements
     with self.command_group('repos', command_type=reposRepositoryOps) as g:
         # repository commands
         g.command('create', 'create_repo', table_transformer=transform_repo_table_output)
@@ -124,3 +130,11 @@ def load_code_commands(self, _):
     with self.command_group('repos import', command_type=reposImportOps) as g:
         # import request
         g.command('create', 'create_import_request', table_transformer=transform_repo_import_table_output)
+
+    with self.command_group('repos ref', command_type=reposRefOps) as g:
+        # refs commands
+        g.command('create', 'create_ref', table_transformer=transform_ref_table_output)
+        g.command('delete', 'delete_ref', table_transformer=transform_ref_table_output)
+        g.command('list', 'list_refs', table_transformer=transform_refs_table_output)
+        g.command('lock', 'lock_ref', table_transformer=transform_ref_table_output)
+        g.command('unlock', 'unlock_ref', table_transformer=transform_ref_table_output)

azure-devops/azext_devops/dev/repos/ref.py

@@ -0,0 +1,131 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from knack.log import get_logger
+from knack.util import CLIError
+from vsts.git.v4_0.models.git_ref_update import GitRefUpdate
+from vsts.exceptions import VstsServiceError
+from azext_devops.dev.common.git import resolve_git_refs
+from azext_devops.dev.common.services import (get_git_client,
+                                              resolve_instance_project_and_repo)
+
+logger = get_logger(__name__)
+
+
+# pylint: disable=redefined-builtin
+def list_refs(filter=None, repository=None, organization=None, project=None, detect=None):
+    """List the references.
+    :param str filter: A filter to apply to the refs (starts with). Example: head or heads/ for the branches.
+    :param str repository: Name or ID of the repository.
+    :param str organization: Azure Devops organization URL. Example: https://dev.azure.com/MyOrganizationName/
+    :param str project: Name or ID of the project.
+    :param str detect: Automatically detect organization and project. Default is "on".
+    """
+    try:
+        organization, project, repository = resolve_instance_project_and_repo(
+            detect=detect,
+            organization=organization,
+            project=project,
+            repo=repository)
+        client = get_git_client(organization)
+        return client.get_refs(repository_id=repository,
+                               project=project,
+                               filter=filter)
+    except VstsServiceError as ex:
+        raise CLIError(ex)
+
+
+def create_ref(name, object_id, repository=None, organization=None, project=None, detect=None):
+    """Create a reference.
+    :param str name: Name of the reference to create (example: heads/my_branch or tags/my_tag).
+    :param str object_id: Id of the object to create the reference from.
+    :param str repository: Name or ID of the repository.
+    :param str organization: Azure Devops organization URL. Example: https://dev.azure.com/MyOrganizationName/
+    :param str project: Name or ID of the project.
+    :param str detect: Automatically detect organization and project. Default is "on".
+    """
+    try:
+        organization, project, repository = resolve_instance_project_and_repo(
+            detect=detect,
+            organization=organization,
+            project=project,
+            repo=repository)
+        client = get_git_client(organization)
+        # by default, the create method does not support setting the is_locked value
+        # to True.
+        ref_update = GitRefUpdate(is_locked=False,
+                                  name=resolve_git_refs(name),
+                                  new_object_id=object_id,
+                                  old_object_id='0000000000000000000000000000000000000000')
+        return client.update_refs(ref_updates=[ref_update],
+                                  repository_id=repository,
+                                  project=project)[0]
+    except VstsServiceError as ex:
+        raise CLIError(ex)
+
+
+def delete_ref(name, object_id, repository=None, organization=None, project=None, detect=None):
+    """Delete a reference.
+    :param str name: Name of the reference to delete (example: heads/my_branch).
+    :param str object_id: Id of the reference to delete.
+    :param str repository: Name or ID of the repository.
+    :param str organization: Azure Devops organization URL. Example: https://dev.azure.com/MyOrganizationName/
+    :param str project: Name or ID of the project.
+    :param str detect: Automatically detect organization and project. Default is "on".
+    """
+    try:
+        organization, project, repository = resolve_instance_project_and_repo(
+            detect=detect,
+            organization=organization,
+            project=project,
+            repo=repository)
+        client = get_git_client(organization)
+        ref_update = GitRefUpdate(name=resolve_git_refs(name),
+                                  new_object_id='0000000000000000000000000000000000000000',
+                                  old_object_id=object_id)
+        return client.update_refs(ref_updates=[ref_update],
+                                  repository_id=repository,
+                                  project=project)[0]
+    except VstsServiceError as ex:
+        raise CLIError(ex)
+
+
+def lock_ref(name, repository=None, organization=None, project=None, detect=None):
+    """Lock a reference.
+    :param str name: Name of the reference to update (example: heads/my_branch).
+    :param str repository: Name or ID of the repository.
+    :param str organization: Azure Devops organization URL. Example: https://dev.azure.com/MyOrganizationName/
+    :param str project: Name or ID of the project.
+    :param str detect: Automatically detect organization and project. Default is "on".
+    """
+    return _update_ref(name, True, repository, organization, project, detect)
+
+
+def unlock_ref(name, repository=None, organization=None, project=None, detect=None):
+    """Unlock a reference.
+    :param str name: Name of the reference to update (example: heads/my_branch).
+    :param str repository: Name or ID of the repository.
+    :param str organization: Azure Devops organization URL. Example: https://dev.azure.com/MyOrganizationName/
+    :param str project: Name or ID of the project.
+    :param str detect: Automatically detect organization and project. Default is "on".
+    """
+    return _update_ref(name, False, repository, organization, project, detect)
+
+
+def _update_ref(name, locked, repository, organization, project, detect):
+    try:
+        organization, project, repository = resolve_instance_project_and_repo(
+            detect=detect,
+            organization=organization,
+            project=project,
+            repo=repository)
+        client = get_git_client(organization)
+        ref_update = GitRefUpdate(is_locked=locked)
+        return client.update_ref(new_ref_info=ref_update,
+                                 repository_id=repository,
+                                 filter=name,
+                                 project=project)
+    except VstsServiceError as ex:
+        raise CLIError(ex)

azure-devops/azext_devops/test/common/_init_.py

@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 import pkg_resources
-pkg_resources.declare_namespace(__name__)
+pkg_resources.declare_namespace(__name__)

azure-devops/azext_devops/test/repos/test_ref.py

@@ -0,0 +1,67 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import unittest
+
+try:
+    # Attempt to load mock (works on Python 3.3 and above)
+    from unittest.mock import patch, ANY
+except ImportError:
+    # Attempt to load mock (works on Python version below 3.3)
+    from mock import patch, ANY
+
+from azext_devops.vstsCompressed.git.v4_0.git_client import GitClient
+from azext_devops.dev.common.services import clear_connection_cache
+from azext_devops.dev.repos.ref import (list_refs, create_ref, delete_ref, lock_ref, unlock_ref)
+from azext_devops.test.utils.helper import get_client_mock_helper, TEST_DEVOPS_ORG_URL
+
+
+class TestRefMethods(unittest.TestCase):
+
+    def setUp(self):
+        self.get_refs_patcher = patch('azext_devops.vstsCompressed.git.v4_0.git_client.GitClient.get_refs')
+        self.update_ref_patcher = patch('azext_devops.vstsCompressed.git.v4_0.git_client.GitClient.update_ref')
+        self.update_refs_patcher = patch('azext_devops.vstsCompressed.git.v4_0.git_client.GitClient.update_refs')
+
+        # patch get client so no network call is made
+        self.get_client = patch('azext_devops.vstsCompressed.vss_connection.VssConnection.get_client', new=get_client_mock_helper)
+        self.resolve_identity_patcher = patch('azext_devops.dev.common.identities.resolve_identity_as_id')
+        self.get_credential_patcher = patch('azext_devops.dev.common.services.get_credential')
+        self.validate_token_patcher = patch('azext_devops.dev.common.services.validate_token_for_instance')
+
+        # start the patchers
+        self.mock_resolve_identity = self.resolve_identity_patcher.start()
+        self.mock_get_credential = self.get_credential_patcher.start()
+        self.mock_validate_token = self.validate_token_patcher.start()
+
+        self.mock_get_refs = self.get_refs_patcher.start()
+        self.mock_update_ref = self.update_ref_patcher.start()
+        self.mock_update_refs = self.update_refs_patcher.start()
+
+        # Setup mocks for clients
+        self.mock_get_client = self.get_client.start()
+
+        # clear connection cache before running each test
+        clear_connection_cache()
+
+    def tearDown(self):
+        self.mock_get_refs.stop()
+        self.mock_update_ref.stop()
+        self.mock_update_refs.stop()
+
+    def test_list_refs(self):
+        # set return values
+        self.mock_validate_token.return_value = True
+
+        response = list_refs(organization=TEST_DEVOPS_ORG_URL,
+                             project='sample_project',
+                             detect='off')
+        self.mock_get_refs.assert_called_once()
+        self.mock_get_refs.assert_called_once_with(filter=None,
+                                                   project='sample_project',
+                                                   repository_id=None)
+
+if __name__ == '__main__':
+    unittest.main()

azure-devops/azext_devops/test/team/_init_.py

@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 import pkg_resources
-pkg_resources.declare_namespace(__name__)
+pkg_resources.declare_namespace(__name__)

doc/authoring_tests.md

@@ -1,4 +1,31 @@
-# Authoring Live Tests
+# Authoring Tests
+
+## Authoring Unit Tests
+
+1. install pytest
+
+1. run all the tests:
+
+    ```bash
+    cd azure-devops
+    pytest
+    ```
+
+1. run an individual test:
+
+    ```bash
+    cd azure-devops/azext_devops
+    pytest test/artifacts/test_universal.py
+    ```
+
+## Authoring Live Tests
+
+1. install `azure-cli-testsdk` and `azure-devtools`:
+
+    ```bash
+    pip install --user 'git+https://github.com/Azure/azure-cli@master#egg=azure-cli-testsdk&subdirectory=src/azure-cli-testsdk' -q
+    pip install --user azure-devtools
+    ```
 
 ## Clear cache while recording a new test