merge llm files
This commit is contained in:
Simon Knott
parent
75c239bf16
commit
47086720c6
|
|
@ -1,195 +0,0 @@
|
||||||
/**
|
|
||||||
* Copyright (c) Microsoft Corporation.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { EventEmitter } from '@testIsomorphic/events';
|
|
||||||
|
|
||||||
export type LLMMessage = {
|
|
||||||
role: 'user' | 'assistant' | 'developer';
|
|
||||||
content: string;
|
|
||||||
displayContent?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface LLM {
|
|
||||||
chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;
|
|
||||||
}
|
|
||||||
|
|
||||||
// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
|
|
||||||
async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {
|
|
||||||
const reader = body.pipeThrough(new TextDecoderStream()).getReader();
|
|
||||||
let buffer = '';
|
|
||||||
|
|
||||||
let lastEventId = '';
|
|
||||||
let type: string = '';
|
|
||||||
let data = '';
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
const { value, done } = await reader.read();
|
|
||||||
if (done)
|
|
||||||
break;
|
|
||||||
buffer += value;
|
|
||||||
const lines = buffer.split('\n');
|
|
||||||
buffer = lines.pop()!; // last line is either empty or incomplete
|
|
||||||
|
|
||||||
for (const line of lines) {
|
|
||||||
if (line.length === 0) {
|
|
||||||
if (data === '') {
|
|
||||||
data = '';
|
|
||||||
type = '';
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (data[data.length - 1] === '\n')
|
|
||||||
data = data.substring(0, data.length - 1);
|
|
||||||
|
|
||||||
const event = { type: type || 'message', data, id: lastEventId };
|
|
||||||
type = '';
|
|
||||||
data = '';
|
|
||||||
|
|
||||||
yield event;
|
|
||||||
}
|
|
||||||
if (line[0] === ':')
|
|
||||||
continue;
|
|
||||||
|
|
||||||
let name = '';
|
|
||||||
let value = '';
|
|
||||||
const colon = line.indexOf(':');
|
|
||||||
if (colon === -1) {
|
|
||||||
name = line;
|
|
||||||
} else {
|
|
||||||
name = line.substring(0, colon);
|
|
||||||
value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (name) {
|
|
||||||
case 'event':
|
|
||||||
type = value;
|
|
||||||
break;
|
|
||||||
case 'data':
|
|
||||||
data += value + '\n';
|
|
||||||
break;
|
|
||||||
case 'id':
|
|
||||||
lastEventId = value;
|
|
||||||
break;
|
|
||||||
case 'retry':
|
|
||||||
default:
|
|
||||||
// not implemented
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class OpenAI implements LLM {
|
|
||||||
|
|
||||||
constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}
|
|
||||||
|
|
||||||
async *chatCompletion(messages: LLMMessage[]) {
|
|
||||||
const url = new URL('./v1/chat/completions', this.baseURL);
|
|
||||||
const response = await fetch(url, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Authorization': `Bearer ${this.apiKey}`,
|
|
||||||
'x-pw-serviceworker': 'forward',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: 'gpt-4o',
|
|
||||||
messages: messages.map(({ role, content }) => ({ role, content })),
|
|
||||||
stream: true,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status !== 200 || !response.body)
|
|
||||||
throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());
|
|
||||||
|
|
||||||
for await (const sseEvent of parseSSE(response.body)) {
|
|
||||||
const event = JSON.parse(sseEvent.data);
|
|
||||||
if (event.object === 'chat.completion.chunk') {
|
|
||||||
if (event.choices[0].finish_reason)
|
|
||||||
break;
|
|
||||||
yield event.choices[0].delta.content;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class Anthropic implements LLM {
|
|
||||||
constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}
|
|
||||||
async *chatCompletion(messages: LLMMessage[]): AsyncGenerator<string> {
|
|
||||||
const response = await fetch(new URL('./v1/messages', this.baseURL), {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'x-api-key': this.apiKey,
|
|
||||||
'anthropic-version': '2023-06-01',
|
|
||||||
'x-pw-serviceworker': 'forward',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: 'claude-3-5-sonnet-20241022',
|
|
||||||
messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),
|
|
||||||
system: messages.find(({ role }) => role === 'developer')?.content,
|
|
||||||
max_tokens: 1024,
|
|
||||||
stream: true,
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status !== 200 || !response.body)
|
|
||||||
throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());
|
|
||||||
|
|
||||||
for await (const sseEvent of parseSSE(response.body)) {
|
|
||||||
const event = JSON.parse(sseEvent.data);
|
|
||||||
if (event.type === 'content_block_delta')
|
|
||||||
yield event.delta.text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class LLMChat {
|
|
||||||
conversations = new Map<string, Conversation>();
|
|
||||||
|
|
||||||
constructor(readonly api: LLM) {}
|
|
||||||
|
|
||||||
getConversation(id: string, systemPrompt: string) {
|
|
||||||
if (!this.conversations.has(id)) {
|
|
||||||
const conversation = new Conversation(this, systemPrompt);
|
|
||||||
this.conversations.set(id, conversation);
|
|
||||||
}
|
|
||||||
return this.conversations.get(id)!;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class Conversation {
|
|
||||||
history: LLMMessage[];
|
|
||||||
onChange = new EventEmitter<void>();
|
|
||||||
|
|
||||||
constructor(private chat: LLMChat, systemPrompt: string) {
|
|
||||||
this.history = [{ role: 'developer', content: systemPrompt }];
|
|
||||||
}
|
|
||||||
|
|
||||||
async send(content: string, displayContent: string | undefined, signal: AbortSignal) {
|
|
||||||
const response: LLMMessage = { role: 'assistant', content: '' };
|
|
||||||
this.history.push({ role: 'user', content, displayContent }, response);
|
|
||||||
this.onChange.fire();
|
|
||||||
for await (const chunk of this.chat.api.chatCompletion(this.history, signal)) {
|
|
||||||
response.content += chunk;
|
|
||||||
this.onChange.fire();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
isEmpty() {
|
|
||||||
return this.history.length === 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,5 +1,200 @@
|
||||||
|
/**
|
||||||
|
* Copyright (c) Microsoft Corporation.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
import * as React from 'react';
|
import * as React from 'react';
|
||||||
import { LLMChat, LLM, OpenAI, Anthropic } from '@isomorphic/llm';
|
import { EventEmitter } from '@testIsomorphic/events';
|
||||||
|
|
||||||
|
export type LLMMessage = {
|
||||||
|
role: 'user' | 'assistant' | 'developer';
|
||||||
|
content: string;
|
||||||
|
displayContent?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
interface LLM {
|
||||||
|
chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
|
||||||
|
async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {
|
||||||
|
const reader = body.pipeThrough(new TextDecoderStream()).getReader();
|
||||||
|
let buffer = '';
|
||||||
|
|
||||||
|
let lastEventId = '';
|
||||||
|
let type: string = '';
|
||||||
|
let data = '';
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const { value, done } = await reader.read();
|
||||||
|
if (done)
|
||||||
|
break;
|
||||||
|
buffer += value;
|
||||||
|
const lines = buffer.split('\n');
|
||||||
|
buffer = lines.pop()!; // last line is either empty or incomplete
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.length === 0) {
|
||||||
|
if (data === '') {
|
||||||
|
data = '';
|
||||||
|
type = '';
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data[data.length - 1] === '\n')
|
||||||
|
data = data.substring(0, data.length - 1);
|
||||||
|
|
||||||
|
const event = { type: type || 'message', data, id: lastEventId };
|
||||||
|
type = '';
|
||||||
|
data = '';
|
||||||
|
|
||||||
|
yield event;
|
||||||
|
}
|
||||||
|
if (line[0] === ':')
|
||||||
|
continue;
|
||||||
|
|
||||||
|
let name = '';
|
||||||
|
let value = '';
|
||||||
|
const colon = line.indexOf(':');
|
||||||
|
if (colon === -1) {
|
||||||
|
name = line;
|
||||||
|
} else {
|
||||||
|
name = line.substring(0, colon);
|
||||||
|
value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (name) {
|
||||||
|
case 'event':
|
||||||
|
type = value;
|
||||||
|
break;
|
||||||
|
case 'data':
|
||||||
|
data += value + '\n';
|
||||||
|
break;
|
||||||
|
case 'id':
|
||||||
|
lastEventId = value;
|
||||||
|
break;
|
||||||
|
case 'retry':
|
||||||
|
default:
|
||||||
|
// not implemented
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class OpenAI implements LLM {
|
||||||
|
|
||||||
|
constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}
|
||||||
|
|
||||||
|
async *chatCompletion(messages: LLMMessage[]) {
|
||||||
|
const url = new URL('./v1/chat/completions', this.baseURL);
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'Authorization': `Bearer ${this.apiKey}`,
|
||||||
|
'x-pw-serviceworker': 'forward',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: 'gpt-4o',
|
||||||
|
messages: messages.map(({ role, content }) => ({ role, content })),
|
||||||
|
stream: true,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status !== 200 || !response.body)
|
||||||
|
throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());
|
||||||
|
|
||||||
|
for await (const sseEvent of parseSSE(response.body)) {
|
||||||
|
const event = JSON.parse(sseEvent.data);
|
||||||
|
if (event.object === 'chat.completion.chunk') {
|
||||||
|
if (event.choices[0].finish_reason)
|
||||||
|
break;
|
||||||
|
yield event.choices[0].delta.content;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Anthropic implements LLM {
|
||||||
|
constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}
|
||||||
|
async *chatCompletion(messages: LLMMessage[]): AsyncGenerator<string> {
|
||||||
|
const response = await fetch(new URL('./v1/messages', this.baseURL), {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'x-api-key': this.apiKey,
|
||||||
|
'anthropic-version': '2023-06-01',
|
||||||
|
'x-pw-serviceworker': 'forward',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: 'claude-3-5-sonnet-20241022',
|
||||||
|
messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),
|
||||||
|
system: messages.find(({ role }) => role === 'developer')?.content,
|
||||||
|
max_tokens: 1024,
|
||||||
|
stream: true,
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status !== 200 || !response.body)
|
||||||
|
throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());
|
||||||
|
|
||||||
|
for await (const sseEvent of parseSSE(response.body)) {
|
||||||
|
const event = JSON.parse(sseEvent.data);
|
||||||
|
if (event.type === 'content_block_delta')
|
||||||
|
yield event.delta.text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class LLMChat {
|
||||||
|
conversations = new Map<string, Conversation>();
|
||||||
|
|
||||||
|
constructor(readonly api: LLM) {}
|
||||||
|
|
||||||
|
getConversation(id: string, systemPrompt: string) {
|
||||||
|
if (!this.conversations.has(id)) {
|
||||||
|
const conversation = new Conversation(this, systemPrompt);
|
||||||
|
this.conversations.set(id, conversation);
|
||||||
|
}
|
||||||
|
return this.conversations.get(id)!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Conversation {
|
||||||
|
history: LLMMessage[];
|
||||||
|
onChange = new EventEmitter<void>();
|
||||||
|
|
||||||
|
constructor(private chat: LLMChat, systemPrompt: string) {
|
||||||
|
this.history = [{ role: 'developer', content: systemPrompt }];
|
||||||
|
}
|
||||||
|
|
||||||
|
async send(content: string, displayContent: string | undefined, signal: AbortSignal) {
|
||||||
|
const response: LLMMessage = { role: 'assistant', content: '' };
|
||||||
|
this.history.push({ role: 'user', content, displayContent }, response);
|
||||||
|
this.onChange.fire();
|
||||||
|
for await (const chunk of this.chat.api.chatCompletion(this.history, signal)) {
|
||||||
|
response.content += chunk;
|
||||||
|
this.onChange.fire();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isEmpty() {
|
||||||
|
return this.history.length < 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
const llmContext = React.createContext<LLMChat | undefined>(undefined);
|
const llmContext = React.createContext<LLMChat | undefined>(undefined);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue