StatefulExecutorBase
Namespace: LLama
The base class for stateful LLama executors.
public abstract class StatefulExecutorBase : LLama.Abstractions.ILLamaExecutor
Inheritance Object → StatefulExecutorBase
Implements ILLamaExecutor
Properties
Model
The mode used by the executor.
public LLamaModel Model { get; }
Property Value
Methods
WithSessionFile(String)
This API is currently not verified.
public StatefulExecutorBase WithSessionFile(string filename)
Parameters
filename String
Returns
Exceptions
SaveSessionFile(String)
This API has not been verified currently.
public void SaveSessionFile(string filename)
Parameters
filename String
HandleRunOutOfContext(Int32)
After running out of the context, take some tokens from the original prompt and recompute the logits in batches.
protected void HandleRunOutOfContext(int tokensToKeep)
Parameters
tokensToKeep Int32
TryReuseMathingPrefix()
Try to reuse the matching prefix from the session file.
protected void TryReuseMathingPrefix()
GetLoopCondition(InferStateArgs)
Decide whether to continue the loop.
protected abstract bool GetLoopCondition(InferStateArgs args)
Parameters
args InferStateArgs
Returns
PreprocessInputs(String, InferStateArgs)
Preprocess the inputs before the inference.
protected abstract void PreprocessInputs(string text, InferStateArgs args)
Parameters
text String
args InferStateArgs
PostProcess(InferenceParams, InferStateArgs, IEnumerable`1&)
Do some post processing after the inference.
protected abstract bool PostProcess(InferenceParams inferenceParams, InferStateArgs args, IEnumerable`1& extraOutputs)
Parameters
inferenceParams InferenceParams
args InferStateArgs
extraOutputs IEnumerable`1&
Returns
InferInternal(InferenceParams, InferStateArgs)
The core inference logic.
protected abstract void InferInternal(InferenceParams inferenceParams, InferStateArgs args)
Parameters
inferenceParams InferenceParams
args InferStateArgs
SaveState(String)
Save the current state to a file.
public abstract void SaveState(string filename)
Parameters
filename String
GetStateData()
Get the current state data.
public abstract ExecutorBaseState GetStateData()
Returns
LoadState(ExecutorBaseState)
Load the state from data.
public abstract void LoadState(ExecutorBaseState data)
Parameters
data ExecutorBaseState
LoadState(String)
Load the state from a file.
public abstract void LoadState(string filename)
Parameters
filename String
Infer(String, InferenceParams, CancellationToken)
Execute the inference.
public IEnumerable<string> Infer(string text, InferenceParams inferenceParams, CancellationToken cancellationToken)
Parameters
text String
inferenceParams InferenceParams
cancellationToken CancellationToken
Returns
InferAsync(String, InferenceParams, CancellationToken)
Execute the inference asynchronously.
public IAsyncEnumerable<string> InferAsync(string text, InferenceParams inferenceParams, CancellationToken cancellationToken)
Parameters
text String
inferenceParams InferenceParams
cancellationToken CancellationToken