Interface ChatCompletionOptions

interface ChatCompletionOptions {
    abortSignal?: AbortSignal;
    nPredict?: number;
    sampling?: SamplingConfig;
    stopTokens?: number[];
    stream?: boolean;
    useCache?: boolean;
    onNewToken(
        token: number,
        piece: Uint8Array,
        currentText: string,
        optionals: { abortSignal: () => any },
    ): any;
}

Properties

abortSignal? nPredict? sampling? stopTokens? stream? useCache?

Methods

onNewToken?

Properties

OptionalabortSignal

abortSignal?: AbortSignal

Optional abort signal to stop the generation. This can also be used to stop during prompt processing (with a bit of delay.)

OptionalnPredict

nPredict?: number

Optionalsampling

sampling?: SamplingConfig

OptionalstopTokens

stopTokens?: number[]

List of custom token IDs for stopping the generation. Note: To convert from text to token ID, use lookupToken()

Optionalstream

stream?: boolean

If true, return an AsyncIterable instead of a string

OptionaluseCache

useCache?: boolean

Equivalent to cache_prompt option in llama.cpp server. Useful for chat, because it skip evaluating the history part of the conversation.

Methods

OptionalonNewToken

  • onNewToken(
        token: number,
        piece: Uint8Array,
        currentText: string,
        optionals: { abortSignal: () => any },
    ): any

  • Parameters

    • token: number
    • piece: Uint8Array
    • currentText: string
    • optionals: { abortSignal: () => any }
      • abortSignal: () => any

        DEPRECATED, use ChatCompletionOptions["abortSignal"] instead

    Returns any

Settings

Member Visibility

On This Page

Properties
Methods