NXDRIVE-903: Renaming folders/files does not sync while offline

Also:
  - Tests: Report is generated on failure only
  - Tests: Less verbosity
  - Packaging: Do not upload to PyPi if Python < 2.7.13 (NXDRIVE-1027)
  - Release 2.5.7

docs/changes.md

@@ -1,9 +1,19 @@
 # dev
 Release date: `2017-??-??`
 
+
+# 2.5.7
+Release date: `2017-11-07`
+
 ### Core
+- [NXDRIVE-903](https://jira.nuxeo.com/browse/NXDRIVE-903): Renaming folders/files does not sync while network interface is OFF
 - [NXDRIVE-1026](https://jira.nuxeo.com/browse/NXDRIVE-1026): Retry in case of connection timeout
 
+#### Minor changes
+- Packaging: Do not upload to PyPi if Python < 2.7.13 (NXDRIVE-1027)
+- Tests: Report is generated on failure only
+- Tests: Less verbosity
+
 
 # 2.5.6
 Release date: `2017-11-02`

docs/technical_changes.md

@@ -3,6 +3,9 @@
 [//]: # (Note 3: keywords ordered [Added, Changed, Moved, Removed])
 
 # dev
+
+
+# 2.5.7
 - Removed `BaseAutomationClient.get_download_buffer()`. Use `FILE_BUFFER_SIZE` attribute instead.
 
 # 2.5.6

nuxeo-drive-client/nxdrive/engine/processor.py

@@ -678,7 +678,8 @@ def _synchronize_locally_deleted(self, doc_pair, local_client, remote_client):
 
     def _synchronize_locally_moved_remotely_modified(self, doc_pair, local_client, remote_client):
         self._synchronize_locally_moved(doc_pair, local_client, remote_client, update=False)
-        self._synchronize_remotely_modified(doc_pair, local_client, remote_client)
+        refreshed_pair = self._dao.get_state_from_id(doc_pair.id)
+        self._synchronize_remotely_modified(refreshed_pair, local_client, remote_client)
 
     def _synchronize_locally_moved_created(self, doc_pair, local_client, remote_client):
         doc_pair.remote_ref = None

nuxeo-drive-client/nxdrive/engine/watcher/remote_watcher.py

@@ -165,6 +165,7 @@ def _check_modified(child_pair, child_info):
             child_pair.remote_can_rename != child_info.can_rename,
             child_pair.remote_can_update != child_info.can_update,
             child_pair.remote_can_create_child != child_info.can_create_child,
+            child_pair.remote_name != child_info.name,
             child_pair.remote_digest != child_info.digest,
         ))
 
@@ -677,9 +678,9 @@ def _update_remote_states(self):
                         # Perform a regular document update on a document
                         # that has been updated, renamed or moved
                         log.debug('Refreshing remote state info for '
-                                  'doc_pair=%r, event_id=%r '
+                                  'doc_pair=%r, event_id=%r, new_info=%r '
                                   '(force_recursion=%d)', doc_pair_repr,
-                                  event_id, event_id == 'securityUpdated')
+                                  event_id, new_info, event_id == 'securityUpdated')
 
                         # Force remote state update in case of a locked / unlocked event since lock info is not
                         # persisted, so not part of the dirty check

nuxeo-drive-client/nxdrive/report.py

@@ -108,7 +108,7 @@ def _export_logs():
     def generate(self):
         """ Create the ZIP report with all interesting files. """
 
-        log.debug('Create report %r', self._report_name)
+        log.debug('Create report %r', self._zipfile)
         log.debug('Manager metrics: %r', self._manager.get_metrics())
         dao = self._manager.get_dao()
         with ZipFile(self._zipfile, mode='w', allowZip64=True) as zip_:

nuxeo-drive-client/tests/common_unit_test.py

@@ -307,6 +307,7 @@ def setUpApp(self, server_profile=None, register_roots=True):
         self.admin_user = os.environ.get('NXDRIVE_TEST_USER', 'Administrator')
         self.password = os.environ.get('NXDRIVE_TEST_PASSWORD', 'Administrator')
         self.build_workspace = os.environ.get('WORKSPACE')
+        self.report_path = os.environ.get('REPORT_PATH')
         self.tearedDown = False
 
         self.tmpdir = None
@@ -611,7 +612,6 @@ def reinit(self):
     def run(self, result=None):
         self.app = StubQApplication([], self)
         self.setUpApp()
-        self.result = result
 
         def launch_test():
             self.root_remote_client.log_on_server(
@@ -633,6 +633,7 @@ def launch_test():
         log.debug('UnitTest run finished')
 
     def tearDown(self):
+        self.generate_report()
         try:
             unittest.TestCase.tearDown(self)
         except StandardError:
@@ -646,9 +647,6 @@ def tearDown(self):
     def tearDownApp(self, server_profile=None):
         if self.tearedDown:
             return
-        if (hasattr(self.result, 'wasSuccessful')
-                and not self.result.wasSuccessful()):
-                self.generate_report()
         log.debug('TearDown unit test')
 
         # Unregister sync roots
@@ -770,13 +768,12 @@ def wait(self, retry=3):
                 self.wait(retry - 1)
 
     def generate_report(self):
-        if "REPORT_PATH" not in os.environ:
+        success = vars(self._resultForDoCleanups).get('_excinfo') is None
+        if success or not self.report_path:
             return
 
-        report_path = os.path.join(os.environ["REPORT_PATH"],
-                                   self.id() + '-' + sys.platform)
-        self.manager_1.generate_report(report_path)
-        log.debug("Report generated in '%s'", report_path)
+        path = os.path.join(self.report_path, self.id() + '-' + sys.platform)
+        self.manager_1.generate_report(path)
 
     def _set_read_permission(self, user, doc_path, grant):
         op_input = "doc:" + doc_path

nuxeo-drive-client/tests/mac_local_client.py

@@ -13,6 +13,11 @@
     import Cocoa
 
 
+class MockFile(object):
+    def __init__(self, path):
+        self.path = path
+
+
 class MacLocalClient(LocalClient):
     def __init__(self, base_folder, **kwargs):
         super(MacLocalClient, self).__init__(base_folder, **kwargs)
@@ -59,6 +64,7 @@ def rename(self, srcref, to_name):
         parent = os.path.dirname(srcref)
         dstref = os.path.join(parent)
         self.move(srcref, dstref, name=to_name)
+        return MockFile(os.path.join(parent, to_name))
 
     def delete(self, ref):
         path = self.abspath(ref)
@@ -69,4 +75,4 @@ def delete(self, ref):
     @staticmethod
     def _process_result(result):
         if not result[0]:
-            raise IOError(result[1].decode('utf-8', 'ignore'), locals())
+            raise IOError(str(result[1]).decode('utf-8', 'ignore'), locals())

nuxeo-drive-client/tests/test_nxdrive_903.py

@@ -0,0 +1,118 @@
+# coding: utf-8
+"""
+ 1 install Nuxeo 7.10
+ 2 install Drive 2.4.6
+ 3 use Administrator account and enable synchronization on Personal Workspace
+ 4 Client 1: configure Drive client to use Administrator user
+ 5 Client 1: create 10 folders
+ 6 Client 1: create 250 files in each folders
+ 7 Client 1: use attached scripts to do it
+ 8 Client 1: wait for synchronization
+ 9 Server: double-check the files are synchronized correctly
+10 Client 2: install Drive 2.4.6 on a second client
+11 Client 2: configure Drive client to use Administrator user
+12 Client 2: wait for synchronization
+13 Client 2: observe the folders and files and synchronized
+14 Client 2: disable the network card (either wifi or cable)
+15 Client 2: wait for Nuxeo client systray icon to switch to grey
+16 Client 1: use the renameFoldersAndFiles.sh in the synchronized folder
+   to rename all the files and folders
+17 Client 1: wait for synchronization
+18 Server: double-check the files and folders are renamed
+19 Client 2: enable network card
+20 Client 2: wait for synchronization
+21 Client 2: only the folders are renamed
+
+This scenario also happens when turning off or exiting Nuxeo Drive client
+instead of disabling the network card.
+"""
+
+from __future__ import unicode_literals
+
+import os.path
+
+from tests.common_unit_test import RandomBug, UnitTestCase
+
+
+class Test(UnitTestCase):
+
+    @RandomBug('NXDRIVE-903', target='windows', mode='BYPASS')
+    def test_nxdrive_903(self):
+        """ On Windows, some files are postponed. Ignore the test if so. """
+
+        remote = self.remote_document_client_1
+        local_1, local_2 = self.local_client_1,  self.local_client_2
+        engine_1, engine_2 = self.engine_1, self.engine_2
+        nb_folders, nb_files = 5, 5
+
+        # Steps 1 -> 13
+        engine_1.start()
+        self.wait_sync(wait_for_async=True)
+
+        folders = [local_1.make_folder('/', 'folder_' + str(idx))
+                   for idx in range(nb_folders)]
+        files = {folder: [local_1.make_file(folder,
+                                            'file_%s_%s.txt' % (num, idx),
+                                            content=bytes(idx))
+                          for idx in range(nb_files)]
+                 for num, folder in enumerate(folders)}
+        self.wait_sync()
+
+        for folder in folders:
+            assert local_1.exists(folder)
+            assert remote.exists(folder)
+            for file_ in files[folder]:
+                assert local_1.exists(file_)
+                assert remote.exists(file_)
+
+        engine_2.start()
+        self.wait_sync(wait_for_async=True,
+                       wait_for_engine_1=False,
+                       wait_for_engine_2=True)
+        for folder in folders:
+            assert local_2.exists(folder)
+            for file_ in files[folder]:
+                assert local_2.exists(file_)
+
+        # Steps 14 -> 15
+        engine_2.suspend()
+
+        # Steps 16 -> 18
+        new_folders = []
+        new_files = {}
+        for folder in folders:
+            name = os.path.basename(folder)
+            new_folder = local_1.rename(folder, name + '-renamed').path
+            new_folders.append(new_folder)
+            new_files[new_folder] = []
+            for file_ in files[folder]:
+                name = os.path.basename(file_)
+                new_name = os.path.splitext(name)[0] + '-renamed.txt'
+                new_file = local_1.rename(new_folder + '/' +  name, new_name)
+                new_files[new_folder].append(new_file.path)
+        self.wait_sync()
+
+        for folder, new_folder in zip(folders, new_folders):
+            assert local_1.exists(new_folder)
+            assert remote.get_info(folder).name == os.path.basename(new_folder)
+            for file_, new_file in zip(files[folder], new_files[new_folder]):
+                assert local_1.exists(new_file)
+                assert remote.get_info(file_).name == os.path.basename(new_file)
+
+        # Steps 19 -> 21
+        engine_2.resume()
+        self.wait_sync(wait_for_async=True, wait_for_engine_2=True)
+
+        # Get a list of all synchronized folders to have a better view of
+        # what is synced as expected
+        states = [local_2.exists(folder) for folder in new_folders]
+        needed = [True] * nb_folders
+        assert states == needed
+
+        # Get a dict of all synchronized files sorted by folders
+        # to have a better view of what is synced as expected
+        states = {folder: [local_2.exists(file_)
+                           for file_ in new_files[folder]]
+                  for folder in new_folders}
+        needed = {folder: [True] * nb_files for folder in new_folders}
+        assert states == needed

nuxeo-drive-client/tests/win_local_client.py

@@ -19,6 +19,11 @@
 from tests.common import log
 
 
+class MockFile(object):
+    def __init__(self, path):
+        self.path = path.replace(os.path.sep, '/')
+
+
 class WindowsLocalClient(LocalClient):
     def __init__(self, base_folder, **kwargs):
         super(WindowsLocalClient, self).__init__(base_folder, **kwargs)
@@ -67,13 +72,15 @@ def abspath(self, ref):
                          'prefix. So the test is likely to fail.'))
         return abs_path[4:]
 
-    def rename(self, ref, to_name):
-        path = self.abspath(ref)
+    def rename(self, srcref, to_name):
+        parent = os.path.dirname(srcref)
+        path = self.abspath(srcref)
         new_path = os.path.join(os.path.dirname(path), to_name)
         res = shell.SHFileOperation((0, shellcon.FO_RENAME, path, new_path,
                                      shellcon.FOF_NOCONFIRMATION, None, None))
         if res[0] != 0:
             raise IOError(res, locals())
+        return MockFile(os.path.join(parent, to_name))
 
     def delete(self, ref):
         path = self.abspath(ref)

tools/posix/deploy_jenkins_slave.sh

@@ -354,7 +354,7 @@ launch_tests() {
         --strict \
         --failed-first \
         -r Efx \
-        -vv
+        -v
 }
 
 start_nxdrive() {

tools/release.sh

@@ -90,10 +90,19 @@ publish_beta() {
 }
 
 publish_on_pip() {
+    local cur_version
     local drive_version
+    local version_ok
+
+    version_ok="$(python -c "import sys; print(sys.version_info > (2, 7, 12))")"
+    if [ "${version_ok}" == "False" ]; then
+        cur_version=$(python --version 2>&1 | awk '{print $2}')
+        echo ">>> Python 2.7.13 or newer is required."
+        echo ">>> Current version is ${cur_version}"
+        return 0
+    fi
 
     drive_version="$(python tools/changelog.py --drive-version)"
-
     echo ">>> [beta ${drive_version}] Creating the virtualenv"
     [ -d pypi ] && rm -rf pypi
     python -m virtualenv pypi

tools/windows/deploy_jenkins_slave.ps1

@@ -330,7 +330,7 @@ function launch_tests {
 		--strict `
 		--failed-first `
 		-r Efx `
-		-vv
+		-v
 	if ($lastExitCode -ne 0) {
 		ExitWithCode $lastExitCode
 	}