IInferenceParams

Namespace: LLama.Abstractions

The parameters used for inference.

public interface IInferenceParams

Properties

TokensKeep

number of tokens to keep from initial prompt

public abstract int TokensKeep { get; set; }

Property Value

Int32

MaxTokens

how many new tokens to predict (n_predict), set to -1 to inifinitely generate response until it complete.

public abstract int MaxTokens { get; set; }

Property Value

Int32

LogitBias

logit bias for specific tokens

public abstract Dictionary<LLamaToken, float> LogitBias { get; set; }

Property Value

Dictionary<LLamaToken, Single>

AntiPrompts

Sequences where the model will stop generating further tokens.

public abstract IReadOnlyList<string> AntiPrompts { get; set; }

Property Value

IReadOnlyList<String>

TopK

0 or lower to use vocab size

public abstract int TopK { get; set; }

Property Value

Int32

TopP

1.0 = disabled

public abstract float TopP { get; set; }

Property Value

Single

MinP

0.0 = disabled

public abstract float MinP { get; set; }

Property Value

Single

TfsZ

1.0 = disabled

public abstract float TfsZ { get; set; }

Property Value

Single

TypicalP

1.0 = disabled

public abstract float TypicalP { get; set; }

Property Value

Single

Temperature

1.0 = disabled

public abstract float Temperature { get; set; }

Property Value

Single

RepeatPenalty

1.0 = disabled

public abstract float RepeatPenalty { get; set; }

Property Value

Single

RepeatLastTokensCount

last n tokens to penalize (0 = disable penalty, -1 = context size) (repeat_last_n)

public abstract int RepeatLastTokensCount { get; set; }

Property Value

Int32

FrequencyPenalty

frequency penalty coefficient 0.0 = disabled

public abstract float FrequencyPenalty { get; set; }

Property Value

Single

PresencePenalty

presence penalty coefficient 0.0 = disabled

public abstract float PresencePenalty { get; set; }

Property Value

Single

Mirostat

Mirostat uses tokens instead of words. algorithm described in the paper https://arxiv.org/abs/2007.14966. 0 = disabled, 1 = mirostat, 2 = mirostat 2.0

public abstract MirostatType Mirostat { get; set; }

Property Value

MirostatType

MirostatTau

target entropy

public abstract float MirostatTau { get; set; }

Property Value

Single

MirostatEta

learning rate

public abstract float MirostatEta { get; set; }

Property Value

Single

PenalizeNL

consider newlines as a repeatable token (penalize_nl)

public abstract bool PenalizeNL { get; set; }

Property Value

Boolean

Grammar

Grammar to constrain possible tokens

public abstract SafeLLamaGrammarHandle Grammar { get; set; }

Property Value

SafeLLamaGrammarHandle

SamplingPipeline

Set a custom sampling pipeline to use. If this is set All other sampling parameters are ignored!

public abstract ISamplingPipeline SamplingPipeline { get; set; }

Property Value

ISamplingPipeline