merge llm files
This commit is contained in:
Simon Knott
parent
75c239bf16
commit
47086720c6
|
|
@ -1,195 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from '@testIsomorphic/events';
|
||||
|
||||
export type LLMMessage = {
|
||||
role: 'user' | 'assistant' | 'developer';
|
||||
content: string;
|
||||
displayContent?: string;
|
||||
};
|
||||
|
||||
export interface LLM {
|
||||
chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;
|
||||
}
|
||||
|
||||
// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
|
||||
async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {
|
||||
const reader = body.pipeThrough(new TextDecoderStream()).getReader();
|
||||
let buffer = '';
|
||||
|
||||
let lastEventId = '';
|
||||
let type: string = '';
|
||||
let data = '';
|
||||
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done)
|
||||
break;
|
||||
buffer += value;
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop()!; // last line is either empty or incomplete
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.length === 0) {
|
||||
if (data === '') {
|
||||
data = '';
|
||||
type = '';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (data[data.length - 1] === '\n')
|
||||
data = data.substring(0, data.length - 1);
|
||||
|
||||
const event = { type: type || 'message', data, id: lastEventId };
|
||||
type = '';
|
||||
data = '';
|
||||
|
||||
yield event;
|
||||
}
|
||||
if (line[0] === ':')
|
||||
continue;
|
||||
|
||||
let name = '';
|
||||
let value = '';
|
||||
const colon = line.indexOf(':');
|
||||
if (colon === -1) {
|
||||
name = line;
|
||||
} else {
|
||||
name = line.substring(0, colon);
|
||||
value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case 'event':
|
||||
type = value;
|
||||
break;
|
||||
case 'data':
|
||||
data += value + '\n';
|
||||
break;
|
||||
case 'id':
|
||||
lastEventId = value;
|
||||
break;
|
||||
case 'retry':
|
||||
default:
|
||||
// not implemented
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class OpenAI implements LLM {
|
||||
|
||||
constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}
|
||||
|
||||
async *chatCompletion(messages: LLMMessage[]) {
|
||||
const url = new URL('./v1/chat/completions', this.baseURL);
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`,
|
||||
'x-pw-serviceworker': 'forward',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'gpt-4o',
|
||||
messages: messages.map(({ role, content }) => ({ role, content })),
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
if (response.status !== 200 || !response.body)
|
||||
throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());
|
||||
|
||||
for await (const sseEvent of parseSSE(response.body)) {
|
||||
const event = JSON.parse(sseEvent.data);
|
||||
if (event.object === 'chat.completion.chunk') {
|
||||
if (event.choices[0].finish_reason)
|
||||
break;
|
||||
yield event.choices[0].delta.content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class Anthropic implements LLM {
|
||||
constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}
|
||||
async *chatCompletion(messages: LLMMessage[]): AsyncGenerator<string> {
|
||||
const response = await fetch(new URL('./v1/messages', this.baseURL), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': this.apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
'x-pw-serviceworker': 'forward',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'claude-3-5-sonnet-20241022',
|
||||
messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),
|
||||
system: messages.find(({ role }) => role === 'developer')?.content,
|
||||
max_tokens: 1024,
|
||||
stream: true,
|
||||
})
|
||||
});
|
||||
|
||||
if (response.status !== 200 || !response.body)
|
||||
throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());
|
||||
|
||||
for await (const sseEvent of parseSSE(response.body)) {
|
||||
const event = JSON.parse(sseEvent.data);
|
||||
if (event.type === 'content_block_delta')
|
||||
yield event.delta.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMChat {
|
||||
conversations = new Map<string, Conversation>();
|
||||
|
||||
constructor(readonly api: LLM) {}
|
||||
|
||||
getConversation(id: string, systemPrompt: string) {
|
||||
if (!this.conversations.has(id)) {
|
||||
const conversation = new Conversation(this, systemPrompt);
|
||||
this.conversations.set(id, conversation);
|
||||
}
|
||||
return this.conversations.get(id)!;
|
||||
}
|
||||
}
|
||||
|
||||
export class Conversation {
|
||||
history: LLMMessage[];
|
||||
onChange = new EventEmitter<void>();
|
||||
|
||||
constructor(private chat: LLMChat, systemPrompt: string) {
|
||||
this.history = [{ role: 'developer', content: systemPrompt }];
|
||||
}
|
||||
|
||||
async send(content: string, displayContent: string | undefined, signal: AbortSignal) {
|
||||
const response: LLMMessage = { role: 'assistant', content: '' };
|
||||
this.history.push({ role: 'user', content, displayContent }, response);
|
||||
this.onChange.fire();
|
||||
for await (const chunk of this.chat.api.chatCompletion(this.history, signal)) {
|
||||
response.content += chunk;
|
||||
this.onChange.fire();
|
||||
}
|
||||
}
|
||||
|
||||
isEmpty() {
|
||||
return this.history.length === 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,200 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
import { LLMChat, LLM, OpenAI, Anthropic } from '@isomorphic/llm';
|
||||
import { EventEmitter } from '@testIsomorphic/events';
|
||||
|
||||
export type LLMMessage = {
|
||||
role: 'user' | 'assistant' | 'developer';
|
||||
content: string;
|
||||
displayContent?: string;
|
||||
};
|
||||
|
||||
interface LLM {
|
||||
chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;
|
||||
}
|
||||
|
||||
// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
|
||||
async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {
|
||||
const reader = body.pipeThrough(new TextDecoderStream()).getReader();
|
||||
let buffer = '';
|
||||
|
||||
let lastEventId = '';
|
||||
let type: string = '';
|
||||
let data = '';
|
||||
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done)
|
||||
break;
|
||||
buffer += value;
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop()!; // last line is either empty or incomplete
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.length === 0) {
|
||||
if (data === '') {
|
||||
data = '';
|
||||
type = '';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (data[data.length - 1] === '\n')
|
||||
data = data.substring(0, data.length - 1);
|
||||
|
||||
const event = { type: type || 'message', data, id: lastEventId };
|
||||
type = '';
|
||||
data = '';
|
||||
|
||||
yield event;
|
||||
}
|
||||
if (line[0] === ':')
|
||||
continue;
|
||||
|
||||
let name = '';
|
||||
let value = '';
|
||||
const colon = line.indexOf(':');
|
||||
if (colon === -1) {
|
||||
name = line;
|
||||
} else {
|
||||
name = line.substring(0, colon);
|
||||
value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case 'event':
|
||||
type = value;
|
||||
break;
|
||||
case 'data':
|
||||
data += value + '\n';
|
||||
break;
|
||||
case 'id':
|
||||
lastEventId = value;
|
||||
break;
|
||||
case 'retry':
|
||||
default:
|
||||
// not implemented
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class OpenAI implements LLM {
|
||||
|
||||
constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}
|
||||
|
||||
async *chatCompletion(messages: LLMMessage[]) {
|
||||
const url = new URL('./v1/chat/completions', this.baseURL);
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.apiKey}`,
|
||||
'x-pw-serviceworker': 'forward',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'gpt-4o',
|
||||
messages: messages.map(({ role, content }) => ({ role, content })),
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
if (response.status !== 200 || !response.body)
|
||||
throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());
|
||||
|
||||
for await (const sseEvent of parseSSE(response.body)) {
|
||||
const event = JSON.parse(sseEvent.data);
|
||||
if (event.object === 'chat.completion.chunk') {
|
||||
if (event.choices[0].finish_reason)
|
||||
break;
|
||||
yield event.choices[0].delta.content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class Anthropic implements LLM {
|
||||
constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}
|
||||
async *chatCompletion(messages: LLMMessage[]): AsyncGenerator<string> {
|
||||
const response = await fetch(new URL('./v1/messages', this.baseURL), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': this.apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
'x-pw-serviceworker': 'forward',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'claude-3-5-sonnet-20241022',
|
||||
messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),
|
||||
system: messages.find(({ role }) => role === 'developer')?.content,
|
||||
max_tokens: 1024,
|
||||
stream: true,
|
||||
})
|
||||
});
|
||||
|
||||
if (response.status !== 200 || !response.body)
|
||||
throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());
|
||||
|
||||
for await (const sseEvent of parseSSE(response.body)) {
|
||||
const event = JSON.parse(sseEvent.data);
|
||||
if (event.type === 'content_block_delta')
|
||||
yield event.delta.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class LLMChat {
|
||||
conversations = new Map<string, Conversation>();
|
||||
|
||||
constructor(readonly api: LLM) {}
|
||||
|
||||
getConversation(id: string, systemPrompt: string) {
|
||||
if (!this.conversations.has(id)) {
|
||||
const conversation = new Conversation(this, systemPrompt);
|
||||
this.conversations.set(id, conversation);
|
||||
}
|
||||
return this.conversations.get(id)!;
|
||||
}
|
||||
}
|
||||
|
||||
class Conversation {
|
||||
history: LLMMessage[];
|
||||
onChange = new EventEmitter<void>();
|
||||
|
||||
constructor(private chat: LLMChat, systemPrompt: string) {
|
||||
this.history = [{ role: 'developer', content: systemPrompt }];
|
||||
}
|
||||
|
||||
async send(content: string, displayContent: string | undefined, signal: AbortSignal) {
|
||||
const response: LLMMessage = { role: 'assistant', content: '' };
|
||||
this.history.push({ role: 'user', content, displayContent }, response);
|
||||
this.onChange.fire();
|
||||
for await (const chunk of this.chat.api.chatCompletion(this.history, signal)) {
|
||||
response.content += chunk;
|
||||
this.onChange.fire();
|
||||
}
|
||||
}
|
||||
|
||||
isEmpty() {
|
||||
return this.history.length < 2;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const llmContext = React.createContext<LLMChat | undefined>(undefined);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue