Module spin_sdk.llm

Module for working with the Spin large language model API

Functions

def generate_embeddings(model: str, text: List[str]) ‑> EmbeddingsResult

Expand source code

def generate_embeddings(model: str, text: List[str]) -> spin_llm.EmbeddingsResult:
    """
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_ModelNotSupported)` will be raised if the component does not have access to the specified model.
    
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_RuntimeError(str))` will be raised if there are any runtime errors.

    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_InvalidInput(str))` will be raised if an invalid input is provided.
    """
    return spin_llm.generate_embeddings(model, text)

A componentize_py_types.Err(Error_ModelNotSupported) will be raised if the component does not have access to the specified model.

A componentize_py_types.Err(Error_RuntimeError(str)) will be raised if there are any runtime errors.

A componentize_py_types.Err(Error_InvalidInput(str)) will be raised if an invalid input is provided.

def infer(model: str, prompt: str) ‑> InferencingResult

Expand source code

def infer(model: str, prompt: str) -> spin_llm.InferencingResult:
    """
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_ModelNotSupported)` will be raised if the component does not have access to the specified model.
    
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_RuntimeError(str))` will be raised if there are any runtime errors.

    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_InvalidInput(str))` will be raised if an invalid input is provided.
    """
    return infer_with_options(model, prompt, None)

A componentize_py_types.Err(Error_ModelNotSupported) will be raised if the component does not have access to the specified model.

A componentize_py_types.Err(Error_RuntimeError(str)) will be raised if there are any runtime errors.

A componentize_py_types.Err(Error_InvalidInput(str)) will be raised if an invalid input is provided.

def infer_with_options(model: str,
prompt: str,
options: InferencingParams | None) ‑> InferencingResult

Expand source code

def infer_with_options(model: str, prompt: str, options: Optional[InferencingParams]) -> spin_llm.InferencingResult:
    """
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_ModelNotSupported)` will be raised if the component does not have access to the specified model.
    
    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_RuntimeError(str))` will be raised if there are any runtime errors.

    A `componentize_py_types.Err(spin_sdk.wit.imports.fermyon_spin_llm_2_0_0.Error_InvalidInput(str))` will be raised if an invalid input is provided.
    """
    some_options = options or InferencingParams()
    my_options = spin_llm.InferencingParams(
        some_options.max_tokens,
        some_options.repeat_penalty,
        some_options.repeat_penalty_last_n_token_count,
        some_options.temperature,
        some_options.top_k,
        some_options.top_p,        
    )
    return spin_llm.infer(model, prompt, my_options)

A componentize_py_types.Err(Error_ModelNotSupported) will be raised if the component does not have access to the specified model.

A componentize_py_types.Err(Error_RuntimeError(str)) will be raised if there are any runtime errors.

A componentize_py_types.Err(Error_InvalidInput(str)) will be raised if an invalid input is provided.

Classes

class InferencingParams (max_tokens: int = 100,
repeat_penalty: float = 1.1,
repeat_penalty_last_n_token_count: int = 64,
temperature: float = 0.8,
top_k: int = 40,
top_p: float = 0.9)

Expand source code

@dataclass
class InferencingParams:
    max_tokens: int = 100
    repeat_penalty: float = 1.1
    repeat_penalty_last_n_token_count: int = 64
    temperature: float = 0.8
    top_k: int = 40
    top_p: float = 0.9

InferencingParams(max_tokens: int = 100, repeat_penalty: float = 1.1, repeat_penalty_last_n_token_count: int = 64, temperature: float = 0.8, top_k: int = 40, top_p: float = 0.9)

Instance variables

var max_tokens : int

var repeat_penalty : float

var repeat_penalty_last_n_token_count : int

var temperature : float

var top_k : int

var top_p : float