feat: Exporting all conversations in one go

api/models/Conversation.js

@@ -75,6 +75,15 @@ module.exports = {
       return { message: 'Error getting conversations' };
     }
   },
+  getAllConvos: async (user) => {
+    try {
+      const convos = await Conversation.find({ user }).sort({ updatedAt: -1 }).lean();
+      return { conversations: convos };
+    } catch (error) {
+      logger.error('[getAllConvos] Error getting conversations', error);
+      return { message: 'Error getting conversations' };
+    }
+  },
   getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 25) => {
     try {
       if (!convoIds || convoIds.length === 0) {

api/server/routes/convos.js

@@ -3,7 +3,10 @@ const express = require('express');
 const { CacheKeys } = require('librechat-data-provider');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { getConvosByPage, deleteConvos, getConvo, saveConvo } = require('~/models/Conversation');
-const { IMPORT_CONVERSATION_JOB_NAME } = require('~/server/utils/import/jobDefinition');
+const {
+  IMPORT_CONVERSATION_JOB_NAME,
+  EXPORT_CONVERSATION_JOB_NAME,
+} = require('~/server/utils/import/jobDefinition');
 const { storage, importFileFilter } = require('~/server/routes/files/multer');
 const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
 const { forkConversation } = require('~/server/utils/import/fork');
@@ -12,6 +15,8 @@ const jobScheduler = require('~/server/utils/jobScheduler');
 const getLogStores = require('~/cache/getLogStores');
 const { sleep } = require('~/server/utils');
 const { logger } = require('~/config');
+const os = require('os');
+const path = require('path');
 
 const router = express.Router();
 router.use(requireJwtAuth);
@@ -168,9 +173,17 @@ router.post('/fork', async (req, res) => {
     res.status(500).send('Error forking conversation');
   }
 });
+router.post('/export', importIpLimiter, importUserLimiter, async (req, res) => {
+  try {
+    const job = await jobScheduler.now(EXPORT_CONVERSATION_JOB_NAME, '', req.user.id);
+    res.status(200).json({ message: 'Export started', jobId: job.id });
+  } catch (error) {
+    console.error('Error exporting conversations', error);
+    res.status(500).send('Error exporting conversations');
+  }
+});
 
-// Get the status of an import job for polling
-router.get('/import/jobs/:jobId', async (req, res) => {
+const jobStatusHandler = async (req, res) => {
   try {
     const { jobId } = req.params;
     const { userId, ...jobStatus } = await jobScheduler.getJobStatus(jobId);
@@ -187,6 +200,33 @@ router.get('/import/jobs/:jobId', async (req, res) => {
     logger.error('Error getting job details', error);
     res.status(500).send('Error getting job details');
   }
+};
+
+// Get the status of an import job for polling
+router.get('/import/jobs/:jobId', jobStatusHandler);
+
+// Get the status of an export job for polling
+router.get('/export/jobs/:jobId', jobStatusHandler);
+
+router.get('/export/jobs/:jobId/conversations.json', async (req, res) => {
+  logger.info('Downloading JSON file');
+  try {
+    const { jobId } = req.params;
+    const tempDir = os.tmpdir();
+    const filePath = path.join(tempDir, `export-${jobId}`);
+
+    res.setHeader('Content-Type', 'application/json');
+
+    res.sendFile(filePath, (err) => {
+      if (err) {
+        console.error(err);
+        res.status(500).send('An error occurred');
+      }
+    });
+  } catch (error) {
+    console.error('Error downloading JSON file', error);
+    res.status(500).send('Error downloading JSON file');
+  }
 });
 
 module.exports = router;

api/server/utils/import/importers.js

@@ -24,12 +24,26 @@ function getImporter(jsonData) {
     return importChatBotUiConvo;
   }
 
-  // For LibreChat
-  if (jsonData.conversationId && (jsonData.messagesTree || jsonData.messages)) {
+  // For LibreChat single conversation
+  if (
+    jsonData.conversationId &&
+    (jsonData.messagesTree || jsonData.messages || jsonData.conversations)
+  ) {
     logger.info('Importing LibreChat conversation');
     return importLibreChatConvo;
   }
 
+  // For LibreChat multiple conversations
+  let firstConvo = jsonData.conversations && jsonData.conversations[0];
+  if (
+    Array.isArray(jsonData.conversations) &&
+    firstConvo?.conversationId &&
+    (firstConvo?.messagesTree || firstConvo?.messages || firstConvo?.conversations)
+  ) {
+    logger.info('Importing multiple LibreChat conversations');
+    return importLibreChatMultiConvo;
+  }
+
   throw new Error('Unsupported import type');
 }
 
@@ -78,11 +92,7 @@ async function importChatBotUiConvo(
  * @param {Function} [builderFactory=createImportBatchBuilder] - The factory function to create an import batch builder.
  * @returns {Promise<void>} - A promise that resolves when the import is complete.
  */
-async function importLibreChatConvo(
-  jsonData,
-  requestUserId,
-  builderFactory = createImportBatchBuilder,
-) {
+async function importLibreChat(jsonData, requestUserId, builderFactory = createImportBatchBuilder) {
   try {
     /** @type {ImportBatchBuilder} */
     const importBatchBuilder = builderFactory(requestUserId);
@@ -179,6 +189,24 @@ async function importLibreChatConvo(
   }
 }
 
+async function importLibreChatConvo(
+  jsonData,
+  requestUserId,
+  builderFactory = createImportBatchBuilder,
+) {
+  return importLibreChat(jsonData, requestUserId, builderFactory);
+}
+
+async function importLibreChatMultiConvo(
+  jsonData,
+  requestUserId,
+  builderFactory = createImportBatchBuilder,
+) {
+  for (const convo of jsonData.conversations) {
+    await importLibreChat(convo, requestUserId, builderFactory);
+  }
+}
+
 /**
  * Imports ChatGPT conversations from provided JSON data.
  * Initializes the import process by creating a batch builder and processing each conversation in the data.

api/server/utils/import/jobDefinition.js

@@ -3,8 +3,13 @@ const jobScheduler = require('~/server/utils/jobScheduler');
 const { getImporter } = require('./importers');
 const { indexSync } = require('~/lib/db');
 const { logger } = require('~/config');
+const { getAllConvos } = require('~/models/Conversation');
+const { getMessages } = require('~/models');
+const os = require('os');
+const path = require('path');
 
 const IMPORT_CONVERSATION_JOB_NAME = 'import conversation';
+const EXPORT_CONVERSATION_JOB_NAME = 'export conversation';
 
 /**
  * Job definition for importing a conversation.
@@ -35,7 +40,67 @@ const importConversationJob = async (job, done) => {
   }
 };
 
-// Call the jobScheduler.define function at startup
+/**
+ * Create a temporary file and delete it after a delay.
+ * @param {object} content - The content to write to the file.
+ * @param {number} delay - The delay in milliseconds to delete the file.
+ * @param {string} job - The job object.
+ * @returns {Promise<string>} The temporary file path.
+ */
+async function createAndDeleteTempFile(content, delay, job) {
+  const { requestUserId } = job.attrs.data;
+  const tempDir = os.tmpdir();
+  const tempFilePath = path.join(tempDir, `export-${job.attrs._id}`);
+  try {
+    await fs.writeFile(tempFilePath, JSON.stringify(content));
+    logger.debug(`user: ${requestUserId} | Created temporary file at: ${tempFilePath}`);
+    setTimeout(async () => {
+      try {
+        await fs.unlink(tempFilePath);
+        logger.debug(
+          `user: ${requestUserId} | Automatically deleted temporary file at: ${tempFilePath}`,
+        );
+      } catch (error) {
+        logger.error(
+          `user: ${requestUserId} | Failed to automatically delete temporary file at: ${tempFilePath}`,
+          error,
+        );
+      }
+    }, delay);
+    return tempFilePath;
+  } catch (error) {
+    logger.error(
+      `user: ${requestUserId} | Error handling the temporary file: ${tempFilePath}`,
+      error,
+    );
+  }
+}
+
+/**
+ * Job definition for exporting all conversations.
+ * @param {import('agenda').Job} job - The job object.
+ * @param {Function} done - The done function.
+ */
+const exportConversationJob = async (job, done) => {
+  const { requestUserId } = job.attrs.data;
+  try {
+    const convos = await getAllConvos(requestUserId);
+
+    for (let i = 0; i < convos.conversations.length; i++) {
+      const conversationId = convos.conversations[i].conversationId;
+      convos.conversations[i].messages = await getMessages({ conversationId });
+    }
+    // Temporary file will be deleted from server after 5 minutes
+    createAndDeleteTempFile(convos, 5 * 60 * 1000, job);
+    done();
+  } catch (error) {
+    logger.error('Failed to export conversation: ', error);
+    done(error);
+  }
+};
+
+// Call the jobScheduler.define functions at startup
 jobScheduler.define(IMPORT_CONVERSATION_JOB_NAME, importConversationJob);
+jobScheduler.define(EXPORT_CONVERSATION_JOB_NAME, exportConversationJob);
 
-module.exports = { IMPORT_CONVERSATION_JOB_NAME };
+module.exports = { IMPORT_CONVERSATION_JOB_NAME, EXPORT_CONVERSATION_JOB_NAME };

client/src/components/Nav/SettingsTabs/Data/Data.tsx

@@ -8,6 +8,7 @@ import { DeleteCacheButton } from './DeleteCacheButton';
 import ImportConversations from './ImportConversations';
 import { ClearChatsButton } from './ClearChats';
 import SharedLinks from './SharedLinks';
+import ExportConversations from './ExportConversations';
 
 function Data() {
   const dataTabRef = useRef(null);
@@ -47,6 +48,9 @@ function Data() {
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
           <ImportConversations />
         </div>
+        <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
+          <ExportConversations />
+        </div>
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
           <SharedLinks />
         </div>

client/src/components/Nav/SettingsTabs/Data/ExportConversations.tsx

@@ -0,0 +1,62 @@
+import React, { useState } from 'react';
+import { ArrowRightFromLine } from 'lucide-react';
+import { useExportConversationsMutation } from '~/data-provider';
+import { useLocalize } from '~/hooks';
+import { useToastContext } from '~/Providers';
+
+function ExportConversations() {
+  const localize = useLocalize();
+  const { showToast } = useToastContext();
+  const [, setErrors] = useState<string[]>([]);
+  const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
+
+  const exportFile = useExportConversationsMutation({
+    onSuccess: (data) => {
+      console.log('Exporting started', data);
+      showToast({ message: localize('com_ui_export_conversation_success') });
+      downloadConversationsJsonFile(data);
+    },
+    onError: (error) => {
+      console.error('Error: ', error);
+      setError(
+        (error as { response: { data: { message?: string } } }).response.data.message ??
+          'An error occurred while exporting the file.',
+      );
+      showToast({ message: localize('com_ui_export_conversation_error'), status: 'error' });
+    },
+  });
+
+  const startExport = () => {
+    const formdata = new FormData();
+    exportFile.mutate(formdata);
+  };
+
+  const downloadConversationsJsonFile = (data) => {
+    const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+    const url = URL.createObjectURL(blob);
+
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = 'conversations.json';
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+  };
+
+  return (
+    <>
+      <div className="flex items-center justify-between">
+        <span>{localize('com_nav_export_all_conversations')}</span>
+        <button
+          onClick={startExport}
+          className="flex h-auto cursor-pointer items-center rounded bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:bg-gray-100 hover:text-green-700 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-green-500"
+        >
+          <ArrowRightFromLine className="mr-1 flex w-[22px] items-center stroke-1" />
+          <span>{localize('com_endpoint_export')}</span>
+        </button>
+      </div>
+    </>
+  );
+}
+
+export default ExportConversations;

client/src/data-provider/mutations.ts

@@ -430,6 +430,71 @@ export const useUploadConversationsMutation = (
   });
 };
 
+export const useExportConversationsMutation = (
+  _options?: t.MutationOptions<t.TImportJobStatus, FormData>,
+) => {
+  const queryClient = useQueryClient();
+  const { onSuccess, onError } = _options || {};
+
+  const checkJobStatus = async (jobId) => {
+    try {
+      const response = await dataService.queryExportAllConversationJobStatus(jobId);
+      return response;
+    } catch (error) {
+      throw new Error('Failed to check job status');
+    }
+  };
+
+  const pollJobStatus = (jobId, onSuccess, onError) => {
+    const intervalId = setInterval(async () => {
+      try {
+        const statusResponse = await checkJobStatus(jobId);
+        console.log('Polling job status:', statusResponse);
+        if (statusResponse.status === 'completed' || statusResponse.status === 'failed') {
+          clearInterval(intervalId);
+          if (statusResponse.status === 'completed') {
+            onSuccess && onSuccess();
+          } else {
+            onError &&
+              onError(new Error(statusResponse.failReason || 'Failed to export conversations'));
+          }
+        }
+      } catch (error) {
+        clearInterval(intervalId);
+        onError && onError(error);
+      }
+    }, 500); // Poll every 0,5 seconds. Adjust time as necessary.
+  };
+
+  return useMutation<t.TImportStartResponse, unknown, FormData>({
+    mutationFn: (formData: FormData) => dataService.exportAllConversationsToJson(formData),
+    onSuccess: (data, variables, context) => {
+      queryClient.invalidateQueries([QueryKeys.allConversations]);
+      const jobId = data.jobId;
+      if (jobId) {
+        console.log('Job ID:', jobId);
+        pollJobStatus(
+          jobId,
+          async () => {
+            queryClient.invalidateQueries([QueryKeys.allConversations]);
+            onSuccess?.(
+              (await dataService.exportConversationsFile(jobId)).data,
+              variables,
+              context,
+            );
+          },
+          (error) => {
+            onError?.(error, variables, { context: 'ExportJobFailed' });
+          },
+        );
+      }
+    },
+    onError: (err, variables, context) => {
+      onError?.(err, variables, context);
+    },
+  });
+};
+
 export const useUploadFileMutation = (
   _options?: t.UploadMutationOptions,
 ): UseMutationResult<