Skip to content

< Back

StatefulExecutorBase

Namespace: LLama

The base class for stateful LLama executors.

1
public abstract class StatefulExecutorBase : LLama.Abstractions.ILLamaExecutor

Inheritance Object → StatefulExecutorBase
Implements ILLamaExecutor
Attributes NullableContextAttribute, NullableAttribute

Fields

_logger

The logger used by this executor.

1
protected ILogger _logger;

_pastTokensCount

The tokens that were already processed by the model.

1
protected int _pastTokensCount;

_consumedTokensCount

The tokens that were consumed by the model during the current inference.

1
protected int _consumedTokensCount;

_n_session_consumed

1
protected int _n_session_consumed;

_n_matching_session_tokens

1
protected int _n_matching_session_tokens;

_pathSession

The path of the session file.

1
protected string _pathSession;

_embeds

A container of the tokens to be processed and after processed.

1
protected List<LLamaToken> _embeds;

_embed_inps

A container for the tokens of input.

1
protected List<LLamaToken> _embed_inps;

_session_tokens

1
protected List<LLamaToken> _session_tokens;

_last_n_tokens

The last tokens generated by the model.

1
protected FixedSizeQueue<LLamaToken> _last_n_tokens;

Properties

Context

The context used by the executor.

1
public LLamaContext Context { get; }

Property Value

LLamaContext

IsMultiModal

1
public bool IsMultiModal { get; }

Property Value

Boolean

ClipModel

1
public LLavaWeights ClipModel { get; }

Property Value

LLavaWeights

Images

1
public List<Byte[]> Images { get; }

Property Value

List<Byte[]>

Constructors

StatefulExecutorBase(LLamaContext, ILogger)

1
protected StatefulExecutorBase(LLamaContext context, ILogger logger)

Parameters

context LLamaContext

logger ILogger

StatefulExecutorBase(LLamaContext, LLavaWeights, ILogger)

1
public StatefulExecutorBase(LLamaContext context, LLavaWeights lLavaWeights, ILogger logger)

Parameters

context LLamaContext

lLavaWeights LLavaWeights

logger ILogger

Methods

WithSessionFile(String)

This API is currently not verified.

1
public StatefulExecutorBase WithSessionFile(string filename)

Parameters

filename String

Returns

StatefulExecutorBase

Exceptions

ArgumentNullException

RuntimeError

SaveSessionFile(String)

This API has not been verified currently.

1
public void SaveSessionFile(string filename)

Parameters

filename String

HandleRunOutOfContext(Int32)

After running out of the context, take some tokens from the original prompt and recompute the logits in batches.

1
protected void HandleRunOutOfContext(int tokensToKeep)

Parameters

tokensToKeep Int32

TryReuseMatchingPrefix()

Try to reuse the matching prefix from the session file.

1
protected void TryReuseMatchingPrefix()

GetLoopCondition(InferStateArgs)

Decide whether to continue the loop.

1
protected abstract Task<bool> GetLoopCondition(InferStateArgs args)

Parameters

args InferStateArgs

Returns

Task<Boolean>

PreprocessInputs(String, InferStateArgs)

Preprocess the inputs before the inference.

1
protected abstract Task PreprocessInputs(string text, InferStateArgs args)

Parameters

text String

args InferStateArgs

Returns

Task

PostProcess(IInferenceParams, InferStateArgs)

Do some post processing after the inference.

1
protected abstract Task<ValueTuple<bool, IReadOnlyList<string>>> PostProcess(IInferenceParams inferenceParams, InferStateArgs args)

Parameters

inferenceParams IInferenceParams

args InferStateArgs

Returns

Task<ValueTuple<Boolean, IReadOnlyList<String>>>

InferInternal(IInferenceParams, InferStateArgs)

The core inference logic.

1
protected abstract Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args)

Parameters

inferenceParams IInferenceParams

args InferStateArgs

Returns

Task

SaveState(String)

Save the current state to a file.

1
public abstract Task SaveState(string filename)

Parameters

filename String

Returns

Task

GetStateData()

Get the current state data.

1
public abstract ExecutorBaseState GetStateData()

Returns

ExecutorBaseState

LoadState(ExecutorBaseState)

Load the state from data.

1
public abstract Task LoadState(ExecutorBaseState data)

Parameters

data ExecutorBaseState

Returns

Task

LoadState(String)

Load the state from a file.

1
public abstract Task LoadState(string filename)

Parameters

filename String

Returns

Task

InferAsync(String, IInferenceParams, CancellationToken)

Execute the inference.

1
public IAsyncEnumerable<string> InferAsync(string text, IInferenceParams inferenceParams, CancellationToken cancellationToken)

Parameters

text String
The prompt. If null, generation will continue where it left off previously.

inferenceParams IInferenceParams

cancellationToken CancellationToken

Returns

IAsyncEnumerable<String>

PrefillPromptAsync(String)

Asynchronously runs a prompt through the model to compute KV cache without generating any new tokens. It could reduce the latency of the first time response if the first input from the user is not immediate.

1
public Task PrefillPromptAsync(string prompt)

Parameters

prompt String
Prompt to process

Returns

Task

< Back