Skip to content

Commit

Permalink
Fix, Improve rendering images in Obsidian, Desktop, Web clients (#659)
Browse files Browse the repository at this point in the history 
- Improve render of inferred query in image chat messages in Web, Desktop apps
- Add inferred queries to image chat responses in Obsidian client
- Fix rendering images from Khoj response in Obsidian client
  • Loading branch information
@debanjum
debanjum committed Feb 24, 2024
2 parents b9d0533 + f846063 commit 170bce2
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 24 deletions.
17 changes: 7 additions & 10 deletions src/interface/desktop/chat.html
View file
Expand Up @@ -199,10 +199,9 @@
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
imageMarkdown += "\n\n";
if (inferredQueries) {
const inferredQuery = inferredQueries?.[0];
imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
const inferredQuery = inferredQueries?.[0];
if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
renderMessage(imageMarkdown, by, dt);
return;
Expand Down Expand Up @@ -392,10 +391,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`;
}
}
if (responseAsJson.detail) {
Expand Down Expand Up @@ -496,10 +494,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
if (responseAsJson.detail) {
Expand Down
32 changes: 26 additions & 6 deletions src/interface/obsidian/src/chat_modal.ts
View file
Expand Up @@ -4,6 +4,8 @@ import { KhojSetting } from 'src/settings';
export interface ChatJsonResult {
image?: string;
detail?: string;
intentType?: string;
inferredQueries?: string[];
}


Expand Down Expand Up @@ -145,11 +147,17 @@ export class KhojChatModal extends Modal {
return referenceButton;
}

renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string) {
renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string, inferredQueries?: string) {
if (!message) {
return;
} else if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:";
for (let inferredQuery of inferredQueries) {
imageMarkdown += `\n\n${inferredQuery}`;
}
}
this.renderMessage(chatEl, imageMarkdown, sender, dt);
return;
} else if (!context) {
Expand Down Expand Up @@ -287,7 +295,15 @@ export class KhojChatModal extends Modal {
} else if (responseJson.response) {
let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response;
chatLogs.forEach((chatLog: any) => {
this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type);
this.renderMessageWithReferences(
chatBodyEl,
chatLog.message,
chatLog.by,
chatLog.context,
new Date(chatLog.created),
chatLog.intent?.type,
chatLog.intent?.["inferred-queries"],
);
});
}
} catch (err) {
Expand Down Expand Up @@ -398,12 +414,16 @@ export class KhojChatModal extends Modal {
// Reset collated chat result to empty string
this.result = "";
responseElement.innerHTML = "";
if (response.headers.get("content-type") == "application/json") {
if (response.headers.get("content-type") === "application/json") {
let responseText = ""
try {
const responseAsJson = await response.json() as ChatJsonResult;
if (responseAsJson.image) {
responseText = `![${query}](data:image/png;base64,${responseAsJson.image})`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
} else if (responseAsJson.detail) {
responseText = responseAsJson.detail;
}
Expand All @@ -413,10 +433,10 @@ export class KhojChatModal extends Modal {
} finally {
await this.renderIncrementalMessage(responseElement, responseText);
}
} else {
// Stream and render chat response
await this.readChatStream(response, responseElement);
}

// Stream and render chat response
await this.readChatStream(response, responseElement);
} catch (err) {
console.log(`Khoj chat response failed with\n${err}`);
let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";
Expand Down
14 changes: 6 additions & 8 deletions src/khoj/interface/web/chat.html
View file
Expand Up @@ -211,10 +211,9 @@
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
imageMarkdown += "\n\n";
if (inferredQueries) {
const inferredQuery = inferredQueries?.[0];
imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
const inferredQuery = inferredQueries?.[0];
if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
renderMessage(imageMarkdown, by, dt);
return;
Expand Down Expand Up @@ -400,10 +399,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
if (responseAsJson.detail) {
Expand Down

0 comments on commit 170bce2

Please sign in to comment.