IJMP-1508: improve content synchronization

zowe · Feb 27, 2024 · 3adaded · 3adaded
1 parent b505210
commit 3adaded
Show file tree

Hide file tree

Showing 18 changed files with 151 additions and 118 deletions.
diff --git a/...in/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/DocumentedSyncProvider.kt b/...in/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/DocumentedSyncProvider.kt
@@ -17,11 +17,14 @@ import com.intellij.openapi.components.service
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.impl.DocumentImpl
 import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.util.text.StringUtil
 import com.intellij.openapi.util.text.StringUtilRt
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.LineSeparator
 import eu.ibagroup.formainframe.dataops.DataOpsManager
 import eu.ibagroup.formainframe.dataops.attributes.RemoteUssAttributes
 import eu.ibagroup.formainframe.utils.castOrNull
+import eu.ibagroup.formainframe.utils.changeEncodingTo
 import eu.ibagroup.formainframe.vfs.MFVirtualFile
 import java.nio.charset.Charset
 import java.util.concurrent.atomic.AtomicBoolean
@@ -88,39 +91,59 @@ class DocumentedSyncProvider(
   override val isReadOnly: Boolean
     get() = getDocument()?.isWritable != true
 
+  /**
+   * Initialize the file properties (charset, line separator)
+   * when the file is opened for the first time.
+   * @param content bytes of the content to init.
+   */
+  private fun initFileProperties(content: ByteArray) {
+    val charset = getCurrentCharset()
+    val text = content.toString(charset)
+    val detectedLineSeparator = runCatching { StringUtil.detectSeparators(text) }.getOrNull() ?: LineSeparator.LF
+    file.detectedLineSeparator = detectedLineSeparator.separatorString
+    changeEncodingTo(file, charset)
+  }
+
   /**
    * Puts initial content in file document.
    * @see SyncProvider.putInitialContent
    */
   override fun putInitialContent(content: ByteArray) {
     if (isInitialContentSet.compareAndSet(false, true)) {
       runCatching {
-        file.getOutputStream(null).use {
-          it.write(content)
-        }
+        initFileProperties(content)
         loadNewContent(content)
       }.onFailure {
         isInitialContentSet.set(false)
       }
     }
   }
 
+  /** Checks if the new content needs to be load */
+  private fun isLoadNeeded(content: ByteArray): Boolean {
+    val currentContent = retrieveCurrentContent()
+    return !content.contentEquals(currentContent)
+  }
+
   /**
    * Update content in file document.
    * @see SyncProvider.loadNewContent
    */
   override fun loadNewContent(content: ByteArray) {
+    if (!isLoadNeeded(content)) return
     val document = getDocument()
     document.castOrNull<DocumentImpl>()?.setAcceptSlashR(true)
     val wasReadOnly = isReadOnly
     if (wasReadOnly) {
       document?.setReadOnly(false)
     }
     val charset = getCurrentCharset()
-    document?.setText(String(content, charset))
+    val text = content.toString(charset)
+    document?.setText(text)
     if (wasReadOnly) {
       document?.setReadOnly(true)
     }
+    saveDocument()
   }
 
   /**

diff --git a/...ibagroup/formainframe/dataops/content/synchronizer/RemoteAttributedContentSynchronizer.kt b/...ibagroup/formainframe/dataops/content/synchronizer/RemoteAttributedContentSynchronizer.kt
@@ -145,22 +145,16 @@ abstract class RemoteAttributedContentSynchronizer<FAttributes : FileAttributes>
       progressIndicator?.text = "Synchronizing file ${syncProvider.file.name} with mainframe"
       val recordId = handlerToStorageIdMap.getOrPut(syncProvider) { successfulStatesStorage.createNewRecord() }
       val attributes = attributesService.getAttributes(syncProvider.file) ?: throw IOException("No Attributes found")
-
       val ussAttributes = attributes.castOrNull<RemoteUssAttributes>()
-      if (!wasFetchedBefore(syncProvider)) {
-        ussAttributes?.let { checkUssFileTag(it) }
-      }
-      val currentCharset = ussAttributes?.charset ?: DEFAULT_TEXT_CHARSET
 
       val fetchedRemoteContentBytes = fetchRemoteContentBytes(attributes, progressIndicator)
       val contentAdapter = dataOpsManager.getMFContentAdapter(syncProvider.file)
       val adaptedFetchedBytes = contentAdapter.adaptContentFromMainframe(fetchedRemoteContentBytes, syncProvider.file)
 
       if (!wasFetchedBefore(syncProvider)) {
         log.info("Setting initial content for file ${syncProvider.file.name}")
+        ussAttributes?.let { checkUssFileTag(it) }
         runWriteActionInEdtAndWait { syncProvider.putInitialContent(adaptedFetchedBytes) }
-        changeFileEncodingTo(syncProvider.file, currentCharset)
-        initLineSeparator(syncProvider)
         successfulStatesStorage.writeStream(recordId).use { it.write(adaptedFetchedBytes) }
         fetchedAtLeastOnce.add(syncProvider)
       } else {

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/SyncAction.kt b/src/main/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/SyncAction.kt
@@ -15,7 +15,6 @@ import com.intellij.openapi.actionSystem.CommonDataKeys
 import com.intellij.openapi.actionSystem.ex.ActionUtil
 import com.intellij.openapi.components.service
 import com.intellij.openapi.editor.ex.EditorEx
-import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.progress.runBackgroundableTask
 import com.intellij.openapi.project.DumbAwareAction
 import com.intellij.openapi.vfs.VirtualFile
@@ -64,15 +63,14 @@ class SyncAction : DumbAwareAction() {
     val vFile = getSupportedVirtualFile(e) ?: return
     val incompatibleEncoding = e.project?.let { !checkEncodingCompatibility(vFile, it) } ?: false
     if (incompatibleEncoding && !showSaveAnywayDialog(vFile.charset)) return
-    val editor = getEditor(e) ?: return
     val syncProvider = DocumentedSyncProvider(vFile, SaveStrategy.default(e.project))
     runBackgroundableTask(
       title = "Synchronizing ${vFile.name}...",
       project = e.project,
       cancellable = true
     ) { indicator ->
       runWriteActionInEdtAndWait {
-        FileDocumentManager.getInstance().saveDocument(editor.document)
+        syncProvider.saveDocument()
         service<DataOpsManager>().getContentSynchronizer(vFile)?.synchronizeWithRemote(syncProvider, indicator)
       }
     }

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/syncUtils.kt b/src/main/kotlin/eu/ibagroup/formainframe/dataops/content/synchronizer/syncUtils.kt
@@ -10,9 +10,6 @@
 
 package eu.ibagroup.formainframe.dataops.content.synchronizer
 
-import com.intellij.openapi.fileEditor.impl.LoadTextUtil
-import com.intellij.openapi.project.Project
-import eu.ibagroup.formainframe.utils.runWriteActionInEdtAndWait
 import java.nio.charset.Charset
 
 private const val NEW_LINE = "\n"
@@ -21,10 +18,6 @@ val DEFAULT_TEXT_CHARSET: Charset = Charset.forName("ISO8859_1")
 
 val DEFAULT_BINARY_CHARSET: Charset = Charset.forName("IBM-1047")
 
-const val LF_LINE_SEPARATOR: String = "\n"
-
-const val CR_LINE_SEPARATOR: String = "\r"
-
 /** Remove string's last blank line */
 fun String.removeLastNewLine(): String {
   return if (endsWith(NEW_LINE)) {
@@ -34,29 +27,20 @@ fun String.removeLastNewLine(): String {
   }
 }
 
+/** Remove last blank line in byte array by converting it to string and back again */
 fun ByteArray.removeLastNewLine(): ByteArray {
-  return if (last() == NEW_LINE.toByte()) {
-    dropLast(1).toByteArray()
-  } else {
-    this
-  }
+  return toString(DEFAULT_TEXT_CHARSET)
+    .removeLastNewLine()
+    .toByteArray(DEFAULT_TEXT_CHARSET)
 }
 
 /** Add new blank line to the string */
 fun ByteArray.addNewLine(): ByteArray {
   return this.plus(NEW_LINE.toByteArray())
 }
 
-/** Initializes the line separator to the contents of the file (by default "\n"). */
-fun initLineSeparator(syncProvider: SyncProvider, project: Project? = null) {
-  val file = syncProvider.file
-  runWriteActionInEdtAndWait {
-    syncProvider.saveDocument()
-    LoadTextUtil.changeLineSeparators(
-      project,
-      file,
-      LoadTextUtil.detectLineSeparator(file, true) ?: LF_LINE_SEPARATOR,
-      file
-    )
-  }
-}
+// TODO: Remove when it becomes possible to mock kotlin inline function.
+/** Wrapper for [String.toByteArray] function. It is necessary only for test purposes for now. */
+fun String.convertToByteArray(charset: Charset = Charsets.UTF_8): ByteArray {
+  return toByteArray(charset)
+}
diff --git a/...n/kotlin/eu/ibagroup/formainframe/dataops/operations/jobs/GetJclRecordsOperationRunner.kt b/...n/kotlin/eu/ibagroup/formainframe/dataops/operations/jobs/GetJclRecordsOperationRunner.kt
@@ -16,6 +16,7 @@ import eu.ibagroup.formainframe.config.connect.ConnectionConfig
 import eu.ibagroup.formainframe.config.connect.authToken
 import eu.ibagroup.formainframe.dataops.DataOpsManager
 import eu.ibagroup.formainframe.dataops.RemoteQuery
+import eu.ibagroup.formainframe.dataops.content.synchronizer.removeLastNewLine
 import eu.ibagroup.formainframe.dataops.exceptions.CallException
 import eu.ibagroup.formainframe.dataops.operations.OperationRunner
 import eu.ibagroup.formainframe.dataops.operations.OperationRunnerFactory
@@ -73,7 +74,7 @@ class GetJclRecordsOperationRunner: OperationRunner<GetJclRecordsOperation, Byte
       }
       else -> throw Exception("Method with such parameters not found")
     }
-    val body = response.body()
+    val body = response.body()?.removeLastNewLine()
     if (!response.isSuccessful || body == null) {
       throw CallException(
         response,

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/dataops/operations/mover/RemoteToLocalFileMover.kt b/src/main/kotlin/eu/ibagroup/formainframe/dataops/operations/mover/RemoteToLocalFileMover.kt
@@ -9,21 +9,17 @@
  */
 package eu.ibagroup.formainframe.dataops.operations.mover
 
-import com.intellij.openapi.fileEditor.impl.LoadTextUtil
 import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry
+import com.intellij.util.LineSeparator
 import eu.ibagroup.formainframe.dataops.DataOpsManager
 import eu.ibagroup.formainframe.dataops.attributes.RemoteUssAttributes
 import eu.ibagroup.formainframe.dataops.content.synchronizer.DocumentedSyncProvider
-import eu.ibagroup.formainframe.dataops.content.synchronizer.LF_LINE_SEPARATOR
 import eu.ibagroup.formainframe.dataops.operations.OperationRunner
 import eu.ibagroup.formainframe.dataops.operations.OperationRunnerFactory
-import eu.ibagroup.formainframe.utils.changeFileEncodingTo
-import eu.ibagroup.formainframe.utils.log
-import eu.ibagroup.formainframe.utils.runReadActionInEdtAndWait
-import eu.ibagroup.formainframe.utils.runWriteActionInEdtAndWait
+import eu.ibagroup.formainframe.utils.*
 import eu.ibagroup.formainframe.vfs.MFVirtualFile
 import org.zowe.kotlinsdk.XIBMDataType
 import java.io.File
@@ -99,10 +95,10 @@ class RemoteToLocalFileMover(val dataOpsManager: DataOpsManager) : AbstractFileM
       }
     }
     val createdFileJava = Paths.get(destFile.path, newFileName).toFile().apply { createNewFile() }
-    createdFileJava.writeBytes(sourceContent)
     if (!sourceFile.fileType.isBinary) {
       setCreatedFileParams(createdFileJava, sourceFile)
     }
+    createdFileJava.writeBytes(sourceContent)
     runReadActionInEdtAndWait {
       destFile.refresh(false, false)
     }
@@ -117,11 +113,9 @@ class RemoteToLocalFileMover(val dataOpsManager: DataOpsManager) : AbstractFileM
   private fun setCreatedFileParams(createdFileJava: File, sourceFile: VirtualFile) {
     val createdVirtualFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(createdFileJava)
     createdVirtualFile?.let {
-      changeFileEncodingTo(it, sourceFile.charset)
-      val lineSeparator = sourceFile.detectedLineSeparator ?: LF_LINE_SEPARATOR
-      runWriteActionInEdtAndWait {
-        LoadTextUtil.changeLineSeparators(null, it, lineSeparator, it)
-      }
+      val lineSeparator = sourceFile.detectedLineSeparator ?: LineSeparator.LF.separatorString
+      it.detectedLineSeparator = lineSeparator
+      runWriteActionInEdtAndWait { changeEncodingTo(it, sourceFile.charset) }
     }
   }
 

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/editor/FileEditorEventsListener.kt b/src/main/kotlin/eu/ibagroup/formainframe/editor/FileEditorEventsListener.kt
@@ -102,7 +102,6 @@ class FileEditorBeforeEventsListener : FileEditorManagerListener.Before {
           if (incompatibleEncoding && !showSaveAnywayDialog(file.charset)) {
             return
           }
-          runWriteActionInEdtAndWait { syncProvider.saveDocument() }
           sendTopic(AutoSyncFileListener.AUTO_SYNC_FILE, project).sync(file)
         }
       }

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/editor/FileEditorFocusListener.kt b/src/main/kotlin/eu/ibagroup/formainframe/editor/FileEditorFocusListener.kt
@@ -71,7 +71,6 @@ class FileEditorFocusListener: FocusChangeListener {
               if (incompatibleEncoding && !showSaveAnywayDialog(file.charset)) {
                 return
               }
-              runWriteActionInEdtAndWait { syncProvider.saveDocument() }
               sendTopic(AutoSyncFileListener.AUTO_SYNC_FILE, project).sync(file)
             }
           }

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/editor/status/MfLineSeparatorPanel.kt b/src/main/kotlin/eu/ibagroup/formainframe/editor/status/MfLineSeparatorPanel.kt
@@ -11,6 +11,7 @@
 package eu.ibagroup.formainframe.editor.status
 
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectManager
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.wm.StatusBar
 import com.intellij.openapi.wm.StatusBarWidget
@@ -51,6 +52,11 @@ class MfLineSeparatorPanel(project: Project): LineSeparatorPanel(project) {
     if (file != null && file.isMfVirtualFile() && !file.isUssVirtualFile()) {
       return false
     }
+    // need to disable changing line separator when more than one project is open
+    // see https://youtrack.jetbrains.com/issue/IDEA-346634/
+    if (ProjectManager.getInstance().openProjects.size > 1) {
+      return false
+    }
     return super.isEnabledForFile(file)
   }
 

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/explorer/actions/EditJclAction.kt b/src/main/kotlin/eu/ibagroup/formainframe/explorer/actions/EditJclAction.kt
@@ -17,15 +17,13 @@ import com.intellij.openapi.fileEditor.OpenFileDescriptor
 import com.intellij.openapi.progress.runBackgroundableTask
 import eu.ibagroup.formainframe.dataops.DataOpsManager
 import eu.ibagroup.formainframe.dataops.attributes.RemoteJobAttributes
-import eu.ibagroup.formainframe.dataops.content.synchronizer.DEFAULT_TEXT_CHARSET
 import eu.ibagroup.formainframe.dataops.content.synchronizer.DocumentedSyncProvider
 import eu.ibagroup.formainframe.dataops.content.synchronizer.SaveStrategy
 import eu.ibagroup.formainframe.dataops.operations.jobs.BasicGetJclRecordsParams
 import eu.ibagroup.formainframe.dataops.operations.jobs.GetJclRecordsOperation
 import eu.ibagroup.formainframe.explorer.ui.JesExplorerView
 import eu.ibagroup.formainframe.explorer.ui.JobNode
 import eu.ibagroup.formainframe.explorer.ui.getExplorerView
-import eu.ibagroup.formainframe.utils.changeFileEncodingTo
 import eu.ibagroup.formainframe.utils.runWriteActionInEdtAndWait
 import eu.ibagroup.formainframe.vfs.MFVirtualFile
 
@@ -84,14 +82,12 @@ class EditJclAction : AnAction() {
                     DocumentedSyncProvider(file = cachedFile, saveStrategy = SaveStrategy.default(e.project))
                   if (!wasCreatedBefore) {
                     syncProvider.putInitialContent(jclContentBytes)
-                    changeFileEncodingTo(cachedFile, DEFAULT_TEXT_CHARSET)
                   } else {
                     val currentContent = syncProvider.retrieveCurrentContent()
                     if (!(currentContent contentEquals jclContentBytes)) {
                       syncProvider.loadNewContent(jclContentBytes)
                     }
                   }
-                  syncProvider.saveDocument()
                   it.navigate(true)
                 }
               }

diff --git a/src/main/kotlin/eu/ibagroup/formainframe/explorer/actions/GetFilePropertiesAction.kt b/src/main/kotlin/eu/ibagroup/formainframe/explorer/actions/GetFilePropertiesAction.kt
@@ -47,6 +47,12 @@ class GetFilePropertiesAction : AnAction() {
           }
 
           is RemoteUssAttributes -> {
+            // TODO: need to think whether this sync is necessary
+            // synchronize charset from file attributes with charset from file properties
+            // if (attributes.charset != virtualFile.charset) {
+            //   attributes.charset = virtualFile.charset
+            //   updateFileTag(attributes)
+            // }
             val oldCharset = attributes.charset
             val initFileMode = attributes.fileMode?.clone()
             val dialog = UssFilePropertiesDialog(e.project, UssFileState(attributes, virtualFile.isBeingEditingNow()))
@@ -75,7 +81,7 @@ class GetFilePropertiesAction : AnAction() {
               }
               val charset = attributes.charset
               if (!virtualFile.isDirectory && oldCharset != charset) {
-                val changed = changeFileEncodingAction(virtualFile, attributes, charset)
+                val changed = changeFileEncodingAction(e.project, virtualFile, attributes, charset)
                 if (!changed) {
                   attributes.charset = oldCharset
                 }