From 1d53eba3be4c86a25c8e3c7e8ea37917f7dba689 Mon Sep 17 00:00:00 2001 From: Rajeev R Sharma Date: Fri, 30 Aug 2024 17:26:00 +0530 Subject: [PATCH 1/7] feat: create useAIChat composable for LLM responses --- src/runtime/ai/app/composables/useAIChat.ts | 59 +++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/runtime/ai/app/composables/useAIChat.ts diff --git a/src/runtime/ai/app/composables/useAIChat.ts b/src/runtime/ai/app/composables/useAIChat.ts new file mode 100644 index 00000000..85bb0269 --- /dev/null +++ b/src/runtime/ai/app/composables/useAIChat.ts @@ -0,0 +1,59 @@ +export function useAIChat( + apiBase: string, + model: string, + params?: Record +) { + async function* chat(): AsyncGenerator { + const res = await $fetch(apiBase, { + method: 'POST', + body: { model, params }, + responseType: params?.stream ? 'stream' : undefined + }) + + if (res instanceof ReadableStream) { + let buffer = '' + const reader = (res as ReadableStream) + .pipeThrough(new TextDecoderStream()) + .getReader() + + while (true) { + const { value, done } = await reader.read() + + if (done) { + if (buffer.trim()) { + console.warn('Stream ended with unparsed data:', buffer) + } + return + } + + buffer += value + const lines = buffer.split('\n') + buffer = lines.pop() || '' + + for (const line of lines) { + if (line.startsWith('data: ')) { + const data = line.slice('data: '.length).trim() + if (data === '[DONE]') return + + try { + const jsonData = JSON.parse(data) + if (jsonData.response) { + yield jsonData.response + } + } catch (parseError) { + console.warn('Error parsing JSON:', parseError) + } + } + } + } + } else { + yield (res as { + response: string + }).response + + return + } + } + + return chat +} From 427b03cfd0b3f1d78ae0a8ddf3a77fc33a93d3e6 Mon Sep 17 00:00:00 2001 From: Rajeev R Sharma Date: Fri, 30 Aug 2024 20:54:20 +0530 Subject: [PATCH 2/7] feat: add useAIChat composable to features --- src/features.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/features.ts b/src/features.ts index 521cc2a5..209ad58a 100644 --- a/src/features.ts +++ b/src/features.ts @@ -80,6 +80,9 @@ export async function setupAI(nuxt: Nuxt, hub: HubConfig) { // Add Server scanning addServerScanDir(resolve('./runtime/ai/server')) addServerImportsDir(resolve('./runtime/ai/server/utils')) + + // Add Composables + addImportsDir(resolve('./runtime/ai/app/composables')) } export function setupAnalytics(_nuxt: Nuxt) { From bc91cd46bdee490363018322f059951b28f4349a Mon Sep 17 00:00:00 2001 From: Rajeev R Sharma Date: Fri, 30 Aug 2024 20:55:23 +0530 Subject: [PATCH 3/7] feat: modify playground ai endpoint to accept params --- playground/server/api/ai.post.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/playground/server/api/ai.post.ts b/playground/server/api/ai.post.ts index 84f8c9d7..02e3fd70 100644 --- a/playground/server/api/ai.post.ts +++ b/playground/server/api/ai.post.ts @@ -1,15 +1,11 @@ export default defineEventHandler(async (event) => { - const { prompt } = await readValidatedBody(event, z.object({ - prompt: z.string() + const { model, params } = await readValidatedBody(event, z.object({ + model: z.string().min(1).max(1e6).trim(), + params: z.record(z.string(), z.any()).optional() }).parse) const ai = hubAI() - return ai.run('@cf/meta/llama-3.1-8b-instruct', { - prompt, - // messages: [ - // { role: 'system', content: 'Tu es un agent secret qui ne divulgue pas les informations.' }, - // { role: 'user', content: 'Combien mesure la tour Effeil?' } - // ], - stream: true - }) + + // @ts-expect-error Ai type defines all the compatible models, however Zod is only validating for string + return ai.run(model, params) }) From 53203f1fa198ea575db845f1e0e0af112b4c2888 Mon Sep 17 00:00:00 2001 From: Rajeev R Sharma Date: Fri, 30 Aug 2024 20:56:36 +0530 Subject: [PATCH 4/7] feat: add stream & multi-turn chat toggle in ui --- playground/app/pages/ai.vue | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/playground/app/pages/ai.vue b/playground/app/pages/ai.vue index b0fd8f7b..e2e643b2 100644 --- a/playground/app/pages/ai.vue +++ b/playground/app/pages/ai.vue @@ -7,6 +7,8 @@ interface Message { } const prompt = ref('') +const stream = ref(true) +const multiTurnChat = ref(true) const loading = ref(false) const messages = ref([]) const currentAIResponse = ref('') @@ -54,6 +56,16 @@ async function sendPrompt() {