Helpers

Helper functions for the usage of the libary.

get_exemplar_selector(name, task, predictor)

Factory function to get an exemplar selector based on the given name.

Parameters:

Name	Type	Description	Default
name	str	The name of the exemplar selector to instantiate.	required
task	BaseTask	The task object to be passed to the selector.	required
predictor	BasePredictor	The predictor object to be passed to the selector.	required

Returns:

Name	Type	Description
BaseExemplarSelector	BaseExemplarSelector	An instance of the requested exemplar selector.

Raises:

Type	Description
ValueError	If the requested selector name is not found.

Source code in promptolution/helpers.py

def get_exemplar_selector(
    name: Literal["random", "random_search"], task: "BaseTask", predictor: "BasePredictor"
) -> "BaseExemplarSelector":
    """Factory function to get an exemplar selector based on the given name.

    Args:
        name (str): The name of the exemplar selector to instantiate.
        task (BaseTask): The task object to be passed to the selector.
        predictor (BasePredictor): The predictor object to be passed to the selector.

    Returns:
        BaseExemplarSelector: An instance of the requested exemplar selector.

    Raises:
        ValueError: If the requested selector name is not found.
    """
    if name == "random_search":
        return RandomSearchSelector(task, predictor)
    elif name == "random":
        return RandomSelector(task, predictor)
    else:
        raise ValueError(f"Unknown exemplar selector: {name}")

get_llm(model_id=None, config=None)

Factory function to create and return a language model instance based on the provided model_id.

This function supports three types of language models: 1. LocalLLM: For running models locally. 2. VLLM: For running models using the vLLM library. 3. APILLM: For API-based models (default if not matching other types).

Parameters:

Name	Type	Description	Default
model_id	str	Identifier for the model to use. Special cases: - "local-{model_name}" for LocalLLM - "vllm-{model_name}" for VLLM - Any other string for APILLM	None
config	ExperimentConfig	"ExperimentConfig" overwriting defaults.	None

Returns:

Type	Description
BaseLLM	An instance of LocalLLM, or APILLM based on the model_id.

Source code in promptolution/helpers.py

def get_llm(model_id: Optional[str] = None, config: Optional["ExperimentConfig"] = None) -> "BaseLLM":
    """Factory function to create and return a language model instance based on the provided model_id.

    This function supports three types of language models:
    1. LocalLLM: For running models locally.
    2. VLLM: For running models using the vLLM library.
    3. APILLM: For API-based models (default if not matching other types).

    Args:
        model_id (str): Identifier for the model to use. Special cases:
                        - "local-{model_name}" for LocalLLM
                        - "vllm-{model_name}" for VLLM
                        - Any other string for APILLM
        config (ExperimentConfig, optional): "ExperimentConfig" overwriting defaults.

    Returns:
        An instance of LocalLLM, or APILLM based on the model_id.
    """
    final_model_id = model_id or (config.model_id if config else None)
    if not final_model_id:
        raise ValueError("model_id must be provided either directly or through config.")

    if "local" in final_model_id:
        model_name = "-".join(final_model_id.split("-")[1:])
        return LocalLLM(model_name, config=config)
    if "vllm" in final_model_id:
        model_name = "-".join(final_model_id.split("-")[1:])
        return VLLM(model_name, config=config)

    return APILLM(model_id=final_model_id, config=config)

get_optimizer(predictor, meta_llm, task, optimizer=None, task_description=None, config=None)

Creates and returns an optimizer instance based on provided parameters.

Parameters:

Name	Type	Description	Default
predictor	BasePredictor	The predictor used for prompt evaluation	required
meta_llm	BaseLLM	The language model used for generating meta-prompts	required
task	BaseTask	The task object used for evaluating prompts	required
optimizer	Optional[OptimizerType]	String identifying which optimizer to use	None
meta_prompt		Meta prompt text for the optimizer	required
task_description	Optional[str]	Description of the task for the optimizer	None
config	Optional[ExperimentConfig]	Configuration object with default parameters	None

Returns:

Type	Description
BaseOptimizer	An optimizer instance

Raises:

Type	Description
ValueError	If an unknown optimizer type is specified

Source code in promptolution/helpers.py

def get_optimizer(
    predictor: "BasePredictor",
    meta_llm: "BaseLLM",
    task: "BaseTask",
    optimizer: Optional["OptimizerType"] = None,
    task_description: Optional[str] = None,
    config: Optional["ExperimentConfig"] = None,
) -> "BaseOptimizer":
    """Creates and returns an optimizer instance based on provided parameters.

    Args:
        predictor: The predictor used for prompt evaluation
        meta_llm: The language model used for generating meta-prompts
        task: The task object used for evaluating prompts
        optimizer: String identifying which optimizer to use
        meta_prompt: Meta prompt text for the optimizer
        task_description: Description of the task for the optimizer
        config: Configuration object with default parameters

    Returns:
        An optimizer instance

    Raises:
        ValueError: If an unknown optimizer type is specified
    """
    final_optimizer = optimizer or (config.optimizer if config else None)
    final_task_description = task_description or (config.task_description if config else None)

    if final_optimizer == "capo":
        crossover_template = (
            CAPO_CROSSOVER_TEMPLATE.replace("<task_desc>", final_task_description)
            if final_task_description
            else CAPO_CROSSOVER_TEMPLATE
        )
        mutation_template = (
            CAPO_MUTATION_TEMPLATE.replace("<task_desc>", final_task_description)
            if final_task_description
            else CAPO_MUTATION_TEMPLATE
        )

        return CAPO(
            predictor=predictor,
            meta_llm=meta_llm,
            task=task,
            crossover_template=crossover_template,
            mutation_template=mutation_template,
            config=config,
        )

    if final_optimizer == "evopromptde":
        template = (
            EVOPROMPT_DE_TEMPLATE_TD.replace("<task_desc>", final_task_description)
            if final_task_description
            else EVOPROMPT_DE_TEMPLATE
        )
        return EvoPromptDE(predictor=predictor, meta_llm=meta_llm, task=task, prompt_template=template, config=config)

    if final_optimizer == "evopromptga":
        template = (
            EVOPROMPT_GA_TEMPLATE_TD.replace("<task_desc>", final_task_description)
            if final_task_description
            else EVOPROMPT_GA_TEMPLATE
        )
        return EvoPromptGA(predictor=predictor, meta_llm=meta_llm, task=task, prompt_template=template, config=config)

    if final_optimizer == "opro":
        template = (
            OPRO_TEMPLATE_TD.replace("<task_desc>", final_task_description) if final_task_description else OPRO_TEMPLATE
        )
        return OPRO(predictor=predictor, meta_llm=meta_llm, task=task, prompt_template=template, config=config)

    raise ValueError(f"Unknown optimizer: {final_optimizer}")

get_predictor(downstream_llm=None, type='marker', *args, **kwargs)

Factory function to create and return a predictor instance.

This function supports three types of predictors: 1. FirstOccurrenceClassifier: A predictor that classifies based on first occurrence of the label. 2. MarkerBasedClassifier: A predictor that classifies based on a marker.

Parameters:

Name	Type	Description	Default
downstream_llm		The language model to use for prediction.	None
type	Literal['first_occurrence', 'marker']	The type of predictor to create: - "first_occurrence" for FirstOccurrenceClassifier - "marker" (default) for MarkerBasedClassifier	'marker'
*args		Variable length argument list passed to the predictor constructor.	()
**kwargs		Arbitrary keyword arguments passed to the predictor constructor.	{}

Returns:

Type	Description
BasePredictor	An instance of FirstOccurrenceClassifier or MarkerBasedClassifier.

Source code in promptolution/helpers.py

def get_predictor(downstream_llm=None, type: "PredictorType" = "marker", *args, **kwargs) -> "BasePredictor":
    """Factory function to create and return a predictor instance.

    This function supports three types of predictors:
    1. FirstOccurrenceClassifier: A predictor that classifies based on first occurrence of the label.
    2. MarkerBasedClassifier: A predictor that classifies based on a marker.

    Args:
        downstream_llm: The language model to use for prediction.
        type (Literal["first_occurrence", "marker"]): The type of predictor to create:
                    - "first_occurrence" for FirstOccurrenceClassifier
                    - "marker" (default) for MarkerBasedClassifier
        *args: Variable length argument list passed to the predictor constructor.
        **kwargs: Arbitrary keyword arguments passed to the predictor constructor.

    Returns:
        An instance of FirstOccurrenceClassifier or MarkerBasedClassifier.
    """
    if type == "first_occurrence":
        return FirstOccurrenceClassifier(downstream_llm, *args, **kwargs)
    elif type == "marker":
        return MarkerBasedClassifier(downstream_llm, *args, **kwargs)
    else:
        raise ValueError(f"Invalid predictor type: '{type}'")

get_task(df, config, task_type=None, judge_llm=None, reward_function=None)

Get the task based on the provided DataFrame and configuration.

So far only ClassificationTask is supported.

Parameters:

Name	Type	Description	Default
df	DataFrame	Input DataFrame containing the data.	required
config	ExperimentConfig	Configuration for the experiment.	required

Returns:

Name	Type	Description
BaseTask	BaseTask	An instance of a task class based on the provided DataFrame and configuration.

Source code in promptolution/helpers.py

def get_task(
    df: pd.DataFrame,
    config: "ExperimentConfig",
    task_type: Optional["TaskType"] = None,
    judge_llm: Optional["BaseLLM"] = None,
    reward_function: Optional[Callable] = None,
) -> "BaseTask":
    """Get the task based on the provided DataFrame and configuration.

    So far only ClassificationTask is supported.

    Args:
        df (pd.DataFrame): Input DataFrame containing the data.
        config (ExperimentConfig): Configuration for the experiment.

    Returns:
        BaseTask: An instance of a task class based on the provided DataFrame and configuration.
    """
    final_task_type = task_type or (config.task_type if config else None)

    if final_task_type == "reward":
        if reward_function is None:
            reward_function = config.reward_function if config else None
        assert reward_function is not None, "Reward function must be provided for reward tasks."
        return RewardTask(
            df=df,
            reward_function=reward_function,
            config=config,
        )
    elif final_task_type == "judge":
        assert judge_llm is not None, "Judge LLM must be provided for judge tasks."
        return JudgeTask(df, judge_llm=judge_llm, config=config)

    return ClassificationTask(df, config=config)

run_evaluation(df, config, prompts)

Run the evaluation phase of the experiment.

Configures all LLMs (downstream, meta, and judge) to use the same instance, that is defined in config.llm.

Parameters:

Name	Type	Description	Default
df	DataFrame	Input DataFrame containing the data.	required
config	Config	Configuration object for the experiment.	required
prompts	List[str]	List of prompts to evaluate.	required

Returns:

Type	Description
DataFrame	pd.DataFrame: A DataFrame containing the prompts and their scores.

Source code in promptolution/helpers.py

def run_evaluation(df: pd.DataFrame, config: "ExperimentConfig", prompts: List[str]) -> pd.DataFrame:
    """Run the evaluation phase of the experiment.

    Configures all LLMs (downstream, meta, and judge) to use
    the same instance, that is defined in `config.llm`.

    Args:
        df (pd.DataFrame): Input DataFrame containing the data.
        config (Config): Configuration object for the experiment.
        prompts (List[str]): List of prompts to evaluate.

    Returns:
        pd.DataFrame: A DataFrame containing the prompts and their scores.
    """
    llm = get_llm(config=config)
    task = get_task(df, config, judge_llm=llm)
    predictor = get_predictor(llm, config=config)
    logger.warning("📊 Starting evaluation...")
    scores = task.evaluate(prompts, predictor, eval_strategy="full")
    df = pd.DataFrame(dict(prompt=prompts, score=scores))
    df = df.sort_values("score", ascending=False, ignore_index=True)

    return df

run_experiment(df, config)

Run a full experiment based on the provided configuration.

Parameters:

Name	Type	Description	Default
df	DataFrame	Input DataFrame containing the data.	required
config	Config	Configuration object for the experiment.	required

Returns:

Type	Description
DataFrame	pd.DataFrame: A DataFrame containing the prompts and their scores.

Source code in promptolution/helpers.py

def run_experiment(df: pd.DataFrame, config: "ExperimentConfig") -> pd.DataFrame:
    """Run a full experiment based on the provided configuration.

    Args:
        df (pd.DataFrame): Input DataFrame containing the data.
        config (Config): Configuration object for the experiment.

    Returns:
        pd.DataFrame: A DataFrame containing the prompts and their scores.
    """
    # train test split
    train_df = df.sample(frac=0.8, random_state=42)
    test_df = df.drop(train_df.index)
    prompts = run_optimization(train_df, config)
    df_prompt_scores = run_evaluation(test_df, config, prompts)

    return df_prompt_scores

run_optimization(df, config)

Run the optimization phase of the experiment.

Configures all LLMs (downstream, meta, and judge) to use the same instance, that is defined in config.llm.

Parameters:

Name	Type	Description	Default
config	Config	Configuration object for the experiment.	required

Returns:

Type	Description
List[str]	List[str]: The optimized list of prompts.

Source code in promptolution/helpers.py

def run_optimization(df: pd.DataFrame, config: "ExperimentConfig") -> List[str]:
    """Run the optimization phase of the experiment.

    Configures all LLMs (downstream, meta, and judge) to use
    the same instance, that is defined in `config.llm`.

    Args:
        config (Config): Configuration object for the experiment.

    Returns:
        List[str]: The optimized list of prompts.
    """
    llm = get_llm(config=config)
    predictor = get_predictor(llm, config=config)

    config.task_description = (config.task_description or "") + " " + (predictor.extraction_description or "")
    if config.optimizer == "capo" and (config.eval_strategy is None or "block" not in config.eval_strategy):
        logger.warning("📌 CAPO requires block evaluation strategy. Setting it to 'sequential_block'.")
        config.eval_strategy = "sequential_block"

    task = get_task(df, config, judge_llm=llm)
    optimizer = get_optimizer(
        predictor=predictor,
        meta_llm=llm,
        task=task,
        config=config,
    )
    logger.warning("🔥 Starting optimization...")
    prompts = optimizer.optimize(n_steps=config.n_steps)

    if hasattr(config, "prepend_exemplars") and config.prepend_exemplars:
        selector = get_exemplar_selector(config.exemplar_selector, task, predictor)
        prompts = [selector.select_exemplars(p, n_examples=config.n_exemplars) for p in prompts]

    return prompts