Mancer support (#562)

common/adapters.ts

@@ -55,6 +55,7 @@ export const AI_ADAPTERS = [
   'goose',
   'replicate',
   'openrouter',
+  'mancer',
 ] as const
 export const CHAT_ADAPTERS = ['default', ...AI_ADAPTERS] as const
 
@@ -197,6 +198,7 @@ export const ADAPTER_LABELS: { [key in AIAdapter]: string } = {
   goose: 'Goose AI',
   replicate: 'Replicate',
   openrouter: 'OpenRouter',
+  mancer: 'Mancer',
 }
 
 export const INSTRUCT_SERVICES: { [key in AIAdapter]?: boolean } = {

common/presets.ts

@@ -299,6 +299,24 @@ export const serviceGenMap: Record<Exclude<ChatAdapter, 'default'>, GenMap> = {
     frequencyPenalty: '',
     gaslight: '',
   },
+  mancer: {
+    maxTokens: 'max_new_tokens',
+    topP: 'top_p',
+    temp: 'temperature',
+    typicalP: 'typical_p',
+    repetitionPenalty: 'repetition_penalty',
+    encoderRepitionPenalty: 'encoder_repetition_penalty',
+    topK: 'top_k',
+    penaltyAlpha: 'penalty_alpha',
+    addBosToken: 'add_bos_token',
+    banEosToken: 'ban_eos_token',
+    skipSpecialTokens: 'skip_special_tokens',
+    topA: '',
+    order: '',
+    repetitionPenaltyRange: '',
+    repetitionPenaltySlope: '',
+    tailFreeSampling: '',
+  },
 }
 
 export function isDefaultPreset(value?: string): value is GenerationPreset {
@@ -336,6 +354,9 @@ export function getFallbackPreset(adapter: AIAdapter): Partial<AppSchema.GenSett
     /** TODO: Create default preset for OpenRouter... */
     case 'openrouter':
       return defaultPresets.openai
+
+    case 'mancer':
+      return defaultPresets.mancer
   }
 }
 
@@ -375,5 +396,8 @@ export function getInferencePreset(
     /** TODO: Create default preset for OpenRouter... */
     case 'openrouter':
       return defaultPresets.openai
+
+    case 'mancer':
+      return defaultPresets.mancer
   }
 }

common/presets/ooba.ts

@@ -15,4 +15,19 @@ export const oobaPresets = {
     typicalP: 1,
     useGaslight: false,
   },
+  mancer: {
+    name: 'Mancer',
+    service: 'mancer',
+    maxTokens: 200,
+    maxContextLength: 8000,
+    repetitionPenalty: 1.2,
+    encoderRepitionPenalty: 1,
+    penaltyAlpha: 0,
+    temp: 0.7,
+    topK: 40,
+    topP: 0.5,
+    topA: 0,
+    typicalP: 1,
+    useGaslight: true,
+  },
 } satisfies Record<string, Partial<AppSchema.GenSettings>>

common/prompt.ts

@@ -774,6 +774,9 @@ export function getContextLimit(
       }
 
       return Math.min(configuredMax, 4096) - genAmount
+
+    case 'mancer':
+      return Math.min(configuredMax, 8000) - genAmount
   }
 }
 

srv/adapter/generate.ts

@@ -27,6 +27,7 @@ import { handleGooseAI } from './goose'
 import { handleReplicate } from './replicate'
 import { getAppConfig } from '../api/settings'
 import { handleOpenRouter } from './openrouter'
+import { handleMancer } from './mancer'
 
 let version = ''
 
@@ -59,6 +60,7 @@ const handlers: { [key in AIAdapter]: ModelAdapter } = {
   goose: handleGooseAI,
   replicate: handleReplicate,
   openrouter: handleOpenRouter,
+  mancer: handleMancer,
 }
 
 type InferenceRequest = {

srv/adapter/mancer.ts

@@ -0,0 +1,137 @@
+import needle from 'needle'
+import { ModelAdapter } from './type'
+import { decryptText } from '../db/util'
+import { sanitise, trimResponseV2 } from '../api/chat/common'
+import { registerAdapter } from './register'
+
+const mancerModels = {
+  'OpenAssistant ORCA': 'https://neuro.mancer.tech/webui/oa-orca/api',
+  'Wizard Vicuna': 'https://neuro.mancer.tech/webui/wizvic/api',
+}
+
+const modelOptions = Object.entries(mancerModels).map(([label, value]) => ({ label, value }))
+
+export const handleMancer: ModelAdapter = async function* (opts) {
+  const body = {
+    prompt: opts.prompt,
+    add_bos_token: opts.gen.addBosToken ?? false,
+    ban_eos_token: opts.gen.banEosToken ?? false,
+    do_sample: true,
+    max_new_tokens: opts.gen.maxTokens,
+    temperature: opts.gen.temp,
+    top_a: opts.gen.topA,
+    top_k: opts.gen.topK,
+    top_p: opts.gen.topP,
+    length_penalty: 1,
+    truncation_length: opts.gen.maxContextLength,
+    typical_p: opts.gen.typicalP,
+    encoder_repetition_penalty: opts.gen.encoderRepitionPenalty,
+    repetition_penalty: opts.gen.repetitionPenalty,
+    repetition_penalty_range: opts.gen.repetitionPenaltyRange,
+    skip_special_tokens: true,
+    tfs: opts.gen.tailFreeSampling,
+    penalty_alpha: opts.gen.penaltyAlpha,
+    num_beams: 1,
+    seed: -1,
+  }
+
+  const url = opts.user.adapterConfig?.mancer?.altUrl || opts.user.adapterConfig?.mancer?.url
+  if (!url) {
+    yield { error: `Mancer request failed: Model/URL not set` }
+    return
+  }
+
+  const key = opts.user.adapterConfig?.mancer?.apiKey
+  if (!key) {
+    yield { error: `Mancer request failed: API key not set` }
+    return
+  }
+
+  opts.log.debug({ ...body, prompt: null }, 'Mancer payload')
+  opts.log.debug(`Prompt:\n${body.prompt}`)
+  yield { prompt: body.prompt }
+
+  const resp = await needle('post', `${url}/v1/generate`, body, {
+    json: true,
+    headers: {
+      'Content-Type': 'application/json',
+      'X-API-KEY': opts.guest ? key : decryptText(key),
+    },
+  }).catch((error) => ({ error }))
+
+  if ('error' in resp) {
+    opts.log.error({ err: resp.error })
+    yield { error: `Mancer request failed: ${resp.error?.message || resp.error}` }
+    return
+  }
+
+  if (resp.statusCode && resp.statusCode >= 400) {
+    opts.log.error({ err: resp.body }, `Mancer request failed {${resp.statusCode}}`)
+    yield {
+      error: `Mancer request failed (${resp.statusCode}) ${
+        resp.body.error || resp.body.message || resp.statusMessage
+      }`,
+    }
+    return
+  }
+
+  try {
+    const text = resp.body.results?.[0]?.text
+    if (!text) {
+      yield {
+        error: `Mancer request failed: Received empty response. Try again.`,
+      }
+      return
+    }
+    yield { meta: { 'credits-spent': resp.body['x-spent-credits'] } }
+    const parsed = sanitise(text.replace(opts.prompt, ''))
+    const trimmed = trimResponseV2(parsed, opts.replyAs, opts.members, opts.characters, [
+      'END_OF_DIALOG',
+    ])
+    yield trimmed || parsed
+  } catch (ex: any) {
+    yield { error: `Mancer request failed: ${ex.message}` }
+    return
+  }
+}
+
+registerAdapter('mancer', handleMancer, {
+  label: 'Mancer',
+  settings: [
+    {
+      field: 'url',
+      label: 'Model',
+      secret: false,
+      setting: { type: 'list', options: modelOptions },
+    },
+    {
+      field: 'urlOverride',
+      label: 'URL Override (See: https://mancer.tech/models.html)',
+      helperText:
+        '(Optional) Overrides the URL from the model selected above - Leave empty if unsure.',
+      secret: false,
+      setting: { type: 'text', placeholder: 'https://neuro.mancer.tech/webui/...../api' },
+    },
+    {
+      field: 'apiKey',
+      label: 'API Key',
+      secret: true,
+      setting: { type: 'text', placeholder: 'E.g. mcr-ahjk7dD2...' },
+    },
+  ],
+  options: [
+    'temp',
+    'addBosToken',
+    'banEosToken',
+    'repetitionPenalty',
+    'repetitionPenaltyRange',
+    'encoderRepitionPenalty',
+    'frequencyPenalty',
+    'gaslight',
+    'topA',
+    'topP',
+    'topK',
+    'typicalP',
+    'penaltyAlpha',
+  ],
+})

srv/adapter/ooba.ts

@@ -48,7 +48,7 @@ export const handleOoba: ModelAdapter = async function* ({
   }).catch((err) => ({ error: err }))
 
   if ('error' in resp) {
-    logger.error({ err: resp.error }, ``)
+    logger.error({ err: resp.error }, `Textgen request failed`)
     yield { error: `Textgen request failed: ${resp.error?.message || resp.error}` }
     return
   }

srv/config.ts

@@ -90,7 +90,7 @@ export const config = {
   },
   adapters: env(
     'ADAPTERS',
-    'novel,horde,kobold,openai,openrouter,scale,claude,ooba,goose,replicate'
+    'novel,horde,kobold,openai,openrouter,scale,claude,ooba,goose,replicate,mancer'
   )
     .split(',')
     .filter((i) => !!i && i in ADAPTER_LABELS) as AIAdapter[],

web/pages/Character/CharacterList.tsx

@@ -319,7 +319,7 @@ const Characters: Component<{
       </Switch>
 
       <Show when={download()}>
-        <DownloadModal show close={() => setDownload()} char={download()!} />
+        <DownloadModal show close={() => setDownload()} charId={download()!._id} />
       </Show>
       <DeleteCharacterModal
         char={showDelete()}

web/pages/Character/CreateCharacterForm.tsx

@@ -690,7 +690,12 @@ export const CreateCharacterForm: Component<{
       </Show>
 
       <Show when={converted()}>
-        <DownloadModal show close={() => setConverted(undefined)} char={converted()!} />
+        <DownloadModal
+          show
+          close={() => setConverted(undefined)}
+          char={converted()!}
+          charId={converted()!._id}
+        />
       </Show>
       <ImportCharacterModal
         show={showImport()}