Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix, Improve rendering images in Obsidian, Desktop, Web clients #659

Merged
merged 3 commits into from Feb 24, 2024
Merged

Fix, Improve rendering images in Obsidian, Desktop, Web clients #659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9b61f0b
Fix rendering images from Khoj response in Obsidian client
debanjum Feb 24, 2024
a2e53d5
Add inferred queries to image chat responses in Obsidian client
debanjum Feb 24, 2024
f846063
Improve render of inferred query in image chat messages in Web, Deskt…
debanjum Feb 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 7 additions & 10 deletions src/interface/desktop/chat.html
View file
Open in desktop
Expand Up @@ -199,10 +199,9 @@
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
imageMarkdown += "\n\n";
if (inferredQueries) {
const inferredQuery = inferredQueries?.[0];
imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
const inferredQuery = inferredQueries?.[0];
if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
renderMessage(imageMarkdown, by, dt);
return;
Expand Down Expand Up @@ -392,10 +391,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`;
}
}
if (responseAsJson.detail) {
Expand Down Expand Up @@ -496,10 +494,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
if (responseAsJson.detail) {
Expand Down
32 changes: 26 additions & 6 deletions src/interface/obsidian/src/chat_modal.ts
View file
Open in desktop
Expand Up @@ -4,6 +4,8 @@ import { KhojSetting } from 'src/settings';
export interface ChatJsonResult {
image?: string;
detail?: string;
intentType?: string;
inferredQueries?: string[];
}


Expand Down Expand Up @@ -145,11 +147,17 @@ export class KhojChatModal extends Modal {
return referenceButton;
}

renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string) {
renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string, inferredQueries?: string) {
if (!message) {
return;
} else if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:";
for (let inferredQuery of inferredQueries) {
imageMarkdown += `\n\n${inferredQuery}`;
}
}
this.renderMessage(chatEl, imageMarkdown, sender, dt);
return;
} else if (!context) {
Expand Down Expand Up @@ -287,7 +295,15 @@ export class KhojChatModal extends Modal {
} else if (responseJson.response) {
let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response;
chatLogs.forEach((chatLog: any) => {
this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type);
this.renderMessageWithReferences(
chatBodyEl,
chatLog.message,
chatLog.by,
chatLog.context,
new Date(chatLog.created),
chatLog.intent?.type,
chatLog.intent?.["inferred-queries"],
);
});
}
} catch (err) {
Expand Down Expand Up @@ -398,12 +414,16 @@ export class KhojChatModal extends Modal {
// Reset collated chat result to empty string
this.result = "";
responseElement.innerHTML = "";
if (response.headers.get("content-type") == "application/json") {
if (response.headers.get("content-type") === "application/json") {
let responseText = ""
try {
const responseAsJson = await response.json() as ChatJsonResult;
if (responseAsJson.image) {
responseText = `![${query}](data:image/png;base64,${responseAsJson.image})`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
} else if (responseAsJson.detail) {
responseText = responseAsJson.detail;
}
Expand All @@ -413,10 +433,10 @@ export class KhojChatModal extends Modal {
} finally {
await this.renderIncrementalMessage(responseElement, responseText);
}
} else {
// Stream and render chat response
await this.readChatStream(response, responseElement);
}

// Stream and render chat response
await this.readChatStream(response, responseElement);
} catch (err) {
console.log(`Khoj chat response failed with\n${err}`);
let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";
Expand Down
14 changes: 6 additions & 8 deletions src/khoj/interface/web/chat.html
View file
Open in desktop
Expand Up @@ -211,10 +211,9 @@
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
if (intentType === "text-to-image") {
let imageMarkdown = `![](data:image/png;base64,${message})`;
imageMarkdown += "\n\n";
if (inferredQueries) {
const inferredQuery = inferredQueries?.[0];
imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
const inferredQuery = inferredQueries?.[0];
if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
renderMessage(imageMarkdown, by, dt);
return;
Expand Down Expand Up @@ -400,10 +399,9 @@
if (responseAsJson.image) {
// If response has image field, response is a generated image.
rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
rawResponse += "\n\n";
const inferredQueries = responseAsJson.inferredQueries?.[0];
if (inferredQueries) {
rawResponse += `**Inferred Query**: ${inferredQueries}`;
const inferredQuery = responseAsJson.inferredQueries?.[0];
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
}
if (responseAsJson.detail) {
Expand Down