chore: improve prompt for PDF with function calling

src/app/modules/form/admin-form/admin-form.assistance.constants.ts

@@ -7,4 +7,23 @@ export const MODEL_TYPE = 'gpt-3.5-turbo'
 
 export const sampleFormFields = `[{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"section"},{"ValidationOptions":{"selectedValidation":null,"customVal":null},"allowPrefill":"<boolean>","lockPrefill":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"textfield"},{"ValidationOptions":{"selectedValidation":null,"customVal":null},"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"textarea"},{"fieldOptions":["<string>"],"othersRadioButton":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"radiobutton"},{"ValidationOptions":{"customMax":null,"customMin":null},"fieldOptions":["<string>"],"othersRadioButton":"<boolean>","validateByValue":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"checkbox"},{"fieldOptions":["<string>"],"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"dropdown"},{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"yes_no"},{"ratingOptions":{"steps":5,"shape":"Star"},"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"rating"},{"autoReplyOptions":{"hasAutoReply":"<boolean>","autoReplySubject":"<string>","autoReplySender":"<string>","autoReplyMessage":"<string>","includeFormSummary":"<boolean>"},"isVerifiable":"<boolean>","hasAllowedEmailDomains":false,"allowedEmailDomains":[],"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"email"},{"allowIntlNumbers":"<boolean>","isVerifiable":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"mobile"},{"allowIntlNumbers":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"homeno"},{"dateValidation":{"customMinDate":null,"customMaxDate":null,"selectedDateValidation":null},"invalidDays":[],"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"date"},{"addMoreRows":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"table","columns":[{"ValidationOptions":{"customVal":null,"selectedValidation":null},"allowPrefill":"<boolean>","lockPrefill":"<boolean>","columnType":"textfield","required":"<boolean>","title":"<string>"}],"minimumRows":2,"maximumRows":null},{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"attachment","attachmentSize":"1"},{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"number","ValidationOptions":{"LengthValidationOptions":{"customVal":null,"selectedLengthValidation":null},"RangeValidationOptions":{"customMin":null,"customMax":null},"selectedValidation":null}},{"ValidationOptions":{"customMax":null,"customMin":null},"validateByValue":"<boolean>","title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"decimal"},{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"nric"},{"title":"<string>","description":"<string>","required":"<boolean>","disabled":"<boolean>","fieldType":"uen"}]`
 
-export const fieldTypes = `'section','textfield','textarea','radiobutton','checkbox','dropdown','yes_no','rating','email','mobile','homeno','date','table','attachment','number','decimal','nric','uen'`
+export const fieldTypes = [
+  'section',
+  'textfield',
+  'textarea',
+  'radiobutton',
+  'checkbox',
+  'dropdown',
+  'yes_no',
+  'rating',
+  'email',
+  'mobile',
+  'homeno',
+  'date',
+  'table',
+  'attachment',
+  'number',
+  'decimal',
+  'nric',
+  'uen',
+]

src/app/modules/form/admin-form/admin-form.assistance.service.ts

@@ -2,7 +2,9 @@ import { AzureKeyCredential, OpenAIClient } from '@azure/openai'
 import {
   ChatRequestMessage,
   ChatResponseMessage,
+  GetChatCompletionsOptions,
 } from '@azure/openai/types/openai'
+// import { FunctionToolDefinition } from '@azure/openai-assistants'
 import { errAsync, okAsync, ResultAsync } from 'neverthrow'
 
 import { ContentTypes } from '../../../../../shared/types/assistance'
@@ -12,6 +14,8 @@ import { createLoggerWithLabel } from '../../../config/logger'
 import { Roles, sampleFormFields } from './admin-form.assistance.constants'
 import {
   formFieldsPromptBuilder,
+  getExpectedQuestionsListTool,
+  getFormFieldsTool,
   isOpenAIError,
   migratePromptBuilder,
   questionListPromptBuilder,
@@ -33,12 +37,22 @@ const azureOpenAi = new OpenAIClient(
   new AzureKeyCredential(azureApiKey),
 )
 
+// const assistantsClient = new AssistantsClient(
+//   endpoint,
+//   new AzureKeyCredential(azureApiKey),
+// )
+
 /**
  * generates a list of questions based on the given type and content
  * @param {string} param.type - The type of content provided. "prompt" or "pdf"
  * @param {string} param.content - prompt or parsed pdf content
  * @returns a ResultAsync containing the generated questions or an AssistanceConnectionError if there was an error connecting to OpenAI
  */
+// export type ChatResponseWithTool = {
+//   message: ChatResponseMessage
+//   tool: FunctionToolDefinition
+// }
+
 export const generateQuestions = ({
   type,
   content,
@@ -66,8 +80,21 @@ export const generateQuestions = ({
       return errAsync(new AssistanceModelTypeError())
   }
 
+  const options = {
+    tools: [
+      {
+        type: 'function',
+        function: getExpectedQuestionsListTool,
+      },
+      // {
+      //   type: 'function',
+      //   function: getFormFieldsTool,
+      // },
+    ],
+  } as GetChatCompletionsOptions
+
   return ResultAsync.fromPromise(
-    azureOpenAi.getChatCompletions(deploymentId, messages),
+    azureOpenAi.getChatCompletions(deploymentId, messages, options),
     (error) => {
       let errorMessage = ''
       // todo: return different error messages based on error codes
@@ -88,6 +115,9 @@ export const generateQuestions = ({
     },
   ).andThen((chatCompletions) => {
     const { message } = chatCompletions.choices[0]
+    console.log('chatCompletions: ', chatCompletions)
+    console.log('messages:', messages)
+    console.log('MESSAGE OBJECT: ', message)
     if (!message) {
       return errAsync(new AssistanceConnectionError())
     }
@@ -116,8 +146,22 @@ export const generateFormFields = (
       content: formFieldsPromptBuilder(questions, sampleFormFields),
     },
   ]
+
+  const options = {
+    tools: [
+      // {
+      //   type: 'function',
+      //   function: getExpectedQuestionsListTool,
+      // },
+      {
+        type: 'function',
+        function: getFormFieldsTool,
+      },
+    ],
+  } as GetChatCompletionsOptions
+
   return ResultAsync.fromPromise(
-    azureOpenAi.getChatCompletions(deploymentId, messages),
+    azureOpenAi.getChatCompletions(deploymentId, messages, options),
     (error) => {
       let errorMessage = ''
       if (isOpenAIError(error)) {
@@ -134,8 +178,10 @@ export const generateFormFields = (
       return new AssistanceConnectionError()
     },
   ).andThen((chatCompletions) => {
+    console.log('chatCompletions form fields: ', chatCompletions)
     const { message } = chatCompletions.choices[0]
     // const {tokenUsage} = chatCompletions.usage?.totalTokens?
+    console.log('form fields message: ', message)
     if (!message) {
       return errAsync(new AssistanceConnectionError())
     }

src/app/modules/form/admin-form/admin-form.assistance.utils.ts

@@ -17,26 +17,108 @@ const expectedQuestionsListFormat =
   '---\n1. <question> | <answer type>\n2. ...\n---'
 
 export const schemaPromptBuilder = (schema: string) => {
-  const prompt = `I am a FormSG, a form builder that has the possible form field schemas in the following list:\n
+  const prompt = `I am a digital, paperless form builder that has the possible form field schemas in the following list:\n
   ${schema}
   Please keep any null values in the schema as null, and false values in the schema as false. Strictly include all keys in the schema, even if they are null or false.`
   return prompt
 }
 
+//Give me a list of content / questions I should have in my form built with this form builder, in the form of "${expectedQuestionsListFormat}". Do not create the question if the <answer type> does not exist in ${fieldTypes}.`
+
 export const questionListPromptBuilder = (purpose: string) => {
-  return `I am a public officer who wants to create a form that collects ${purpose}.
-  Give me a list of content / questions I should have in my form built with this form builder, in the form of "${expectedQuestionsListFormat}", where <answer type> must follow the category of types within ${fieldTypes}. Do not create the question if the <answer type> does not exist in ${fieldTypes}.`
+  return `I am an administrator who wants to create a digital form that collects ${purpose}.`
+}
+
+export const getExpectedQuestionsListTool = {
+  name: 'getExpectedQuestions',
+  description: `Gets a list of questions to build a form with the specified answerTypes. Ensure all answerTypes exist in the enum. Do not create a question if the answer type does not exist. Signatures should be treated as textfield because an answerType of signature cannot not exist in digital forms.
+    Examples:
+      1. What is your name? | textfield
+      2. Applicant/Requestor Signature: | textfield`,
+  parameters: {
+    type: 'object',
+    properties: {
+      questionListFormat: {
+        type: 'string',
+        format: '---\n1. question | answerType\n2. ...\n---',
+      },
+      answerType: {
+        type: 'string',
+        enum: [
+          'section',
+          'textfield',
+          'textarea',
+          'radiobutton',
+          'checkbox',
+          'dropdown',
+          'yes_no',
+          'rating',
+          'email',
+          'mobile',
+          'homeno',
+          'date',
+          'table',
+          'attachment',
+          'number',
+          'decimal',
+          'nric',
+          'uen',
+        ],
+      },
+    },
+    required: ['questionListFormat', 'answerType'],
+  },
 }
 
 export const formFieldsPromptBuilder = (questions: string, schema: string) => {
-  return `Help me generate a form with the following list of questions: ${questions}
-  Provide the questions as FormSG form fields in JSON format (with the following keys: ${schema}), in the form of "${expectedFormFieldSchemaFormat}" as defined by the system, without any code blocks. Format the JSON as a single line. Ensure the JSON generated only contain fieldTypes of types ${fieldTypes}. Do not create any fieldTypes which are not ${fieldTypes}. Replace values in <> with actual primitive values. Do not build the fieldType if a path required is not available.`
+  return `Help me generate a digital form with the following list of questions: ${questions}
+  Provide the questions as form fields for a digital form in JSON format (with the following keys: ${schema}), in the form of "${expectedFormFieldSchemaFormat}" as defined by the system, without any code blocks. Format the JSON as a single line. Ensure the JSON generated only contain fieldTypes of types ${fieldTypes}. Do not create any fieldTypes which are not ${fieldTypes}. Replace values in <> with actual primitive values. Do not build the fieldType if a path required is not available.`
 }
+
+export const getFormFieldsTool = {
+  name: 'getFormFields',
+  description:
+    'Gets form fields to build a form with the specified field types. Do not create fields which do not belong to any of the specified field types. Signatures should be treated as textfield because a field type of signature cannot not exist in digital forms.',
+  parameters: {
+    type: 'object',
+    properties: {
+      formFieldsFormat: {
+        type: 'string',
+        format: '---\nJSON schema; array of form fields\n---',
+      },
+      fieldTypes: {
+        type: 'string',
+        enum: [
+          'section',
+          'textfield',
+          'textarea',
+          'radiobutton',
+          'checkbox',
+          'dropdown',
+          'yes_no',
+          'rating',
+          'email',
+          'mobile',
+          'homeno',
+          'date',
+          'table',
+          'attachment',
+          'number',
+          'decimal',
+          'nric',
+          'uen',
+        ],
+      },
+    },
+    required: ['formFieldsFormat', 'fieldType'],
+  },
+}
+
 export const migratePromptBuilder = (parsedContent: string) => {
   return `Help me generate the corresponding JSON form fields from content parsed from a PDF document.
   Here is the parsed content from the PDF document (wrapped in triple quotes):
   """
   ${parsedContent}
   """
-  Based on the parsed content, extract content that should be added to the form builder form and present them as a list, in the form of "${expectedQuestionsListFormat}".`
+  Based on the parsed content, extract content that should be added to the form builder form and present them as a list, in the form of "${expectedQuestionsListFormat}". Replace values in <> with actual primitive values.`
 }