Exemplar Selectors

Module for exemplar selectors.

base_exemplar_selector

Base class for exemplar selectors.

BaseExemplarSelector

Bases: ABC

An abstract base class for exemplar selectors.

This class defines the basic interface and common functionality that all exemplar selectors should implement.

Source code in promptolution/exemplar_selectors/base_exemplar_selector.py

class BaseExemplarSelector(ABC):
    """An abstract base class for exemplar selectors.

    This class defines the basic interface and common functionality
    that all exemplar selectors should implement.
    """

    def __init__(self, task: "BaseTask", predictor: "BasePredictor", config: "ExperimentConfig" = None):
        """Initialize the BaseExemplarSelector.

        Args:
            task (BaseTask): An object representing the task to be performed.
            predictor (BasePredictor): An object capable of making predictions based on prompts.
            config (ExperimentConfig, optional): "ExperimentConfig" overwriting the defaults
        """
        self.task = task
        self.predictor = predictor

        if config is not None:
            config.apply_to(self)

    @abstractmethod
    def select_exemplars(self, prompt: str, n_examples: int = 5) -> str:
        """Select exemplars based on the given prompt.

        Args:
            prompt (str): The input prompt to base the exemplar selection on.
            n_examples (int, optional): The number of exemplars to select. Defaults to 5.

        Returns:
            str: A new prompt that includes the original prompt and the selected exemplars.

        Raises:
            NotImplementedError: This method should be implemented by subclasses.
        """
        raise NotImplementedError("This method should be implemented by subclasses.")

__init__(task, predictor, config=None)

Initialize the BaseExemplarSelector.

Parameters:

Name	Type	Description	Default
task	BaseTask	An object representing the task to be performed.	required
predictor	BasePredictor	An object capable of making predictions based on prompts.	required
config	ExperimentConfig	"ExperimentConfig" overwriting the defaults	None

Source code in promptolution/exemplar_selectors/base_exemplar_selector.py

def __init__(self, task: "BaseTask", predictor: "BasePredictor", config: "ExperimentConfig" = None):
    """Initialize the BaseExemplarSelector.

    Args:
        task (BaseTask): An object representing the task to be performed.
        predictor (BasePredictor): An object capable of making predictions based on prompts.
        config (ExperimentConfig, optional): "ExperimentConfig" overwriting the defaults
    """
    self.task = task
    self.predictor = predictor

    if config is not None:
        config.apply_to(self)

select_exemplars(prompt, n_examples=5) abstractmethod

Select exemplars based on the given prompt.

Parameters:

Name	Type	Description	Default
prompt	str	The input prompt to base the exemplar selection on.	required
n_examples	int	The number of exemplars to select. Defaults to 5.	5

Returns:

Name	Type	Description
str	str	A new prompt that includes the original prompt and the selected exemplars.

Raises:

Type	Description
NotImplementedError	This method should be implemented by subclasses.

Source code in promptolution/exemplar_selectors/base_exemplar_selector.py

@abstractmethod
def select_exemplars(self, prompt: str, n_examples: int = 5) -> str:
    """Select exemplars based on the given prompt.

    Args:
        prompt (str): The input prompt to base the exemplar selection on.
        n_examples (int, optional): The number of exemplars to select. Defaults to 5.

    Returns:
        str: A new prompt that includes the original prompt and the selected exemplars.

    Raises:
        NotImplementedError: This method should be implemented by subclasses.
    """
    raise NotImplementedError("This method should be implemented by subclasses.")

random_search_selector

Random search exemplar selector.

RandomSearchSelector

Bases: BaseExemplarSelector

A selector that uses random search to find the best set of exemplars.

This class implements a strategy that generates multiple sets of random examples, evaluates their performance, and selects the best performing set.

Source code in promptolution/exemplar_selectors/random_search_selector.py

class RandomSearchSelector(BaseExemplarSelector):
    """A selector that uses random search to find the best set of exemplars.

    This class implements a strategy that generates multiple sets of random examples,
    evaluates their performance, and selects the best performing set.
    """

    def select_exemplars(self, prompt, n_examples: int = 5, n_trials: int = 5):
        """Select exemplars using a random search strategy.

        This method generates multiple sets of random examples, evaluates their performance
        when combined with the original prompt, and returns the best performing set.

        Args:
            prompt (str): The input prompt to base the exemplar selection on.
            n_examples (int, optional): The number of exemplars to select in each trial. Defaults to 5.
            n_trials (int, optional): The number of random trials to perform. Defaults to 5.

        Returns:
            str: The best performing prompt, which includes the original prompt and the selected exemplars.
        """
        best_score = 0
        best_prompt = prompt

        for _ in range(n_trials):
            _, seq = self.task.evaluate(prompt, self.predictor, n_samples=n_examples, subsample=True, return_seq=True)
            prompt_with_examples = "\n\n".join([prompt] + seq) + "\n\n"
            # evaluate prompts as few shot prompt
            score = self.task.evaluate(prompt_with_examples, self.predictor, subsample=True)
            if score > best_score:
                best_score = score
                best_prompt = prompt_with_examples

        return best_prompt

select_exemplars(prompt, n_examples=5, n_trials=5)

Select exemplars using a random search strategy.

This method generates multiple sets of random examples, evaluates their performance when combined with the original prompt, and returns the best performing set.

Parameters:

Name	Type	Description	Default
prompt	str	The input prompt to base the exemplar selection on.	required
n_examples	int	The number of exemplars to select in each trial. Defaults to 5.	5
n_trials	int	The number of random trials to perform. Defaults to 5.	5

Returns:

Name	Type	Description
str		The best performing prompt, which includes the original prompt and the selected exemplars.

Source code in promptolution/exemplar_selectors/random_search_selector.py

def select_exemplars(self, prompt, n_examples: int = 5, n_trials: int = 5):
    """Select exemplars using a random search strategy.

    This method generates multiple sets of random examples, evaluates their performance
    when combined with the original prompt, and returns the best performing set.

    Args:
        prompt (str): The input prompt to base the exemplar selection on.
        n_examples (int, optional): The number of exemplars to select in each trial. Defaults to 5.
        n_trials (int, optional): The number of random trials to perform. Defaults to 5.

    Returns:
        str: The best performing prompt, which includes the original prompt and the selected exemplars.
    """
    best_score = 0
    best_prompt = prompt

    for _ in range(n_trials):
        _, seq = self.task.evaluate(prompt, self.predictor, n_samples=n_examples, subsample=True, return_seq=True)
        prompt_with_examples = "\n\n".join([prompt] + seq) + "\n\n"
        # evaluate prompts as few shot prompt
        score = self.task.evaluate(prompt_with_examples, self.predictor, subsample=True)
        if score > best_score:
            best_score = score
            best_prompt = prompt_with_examples

    return best_prompt

random_selector

Random exemplar selector.

RandomSelector

Bases: BaseExemplarSelector

A selector that randomly selects correct exemplars.

This class implements a strategy that generates random examples and selects those that are evaluated as correct until the desired number of exemplars is reached.

Source code in promptolution/exemplar_selectors/random_selector.py

class RandomSelector(BaseExemplarSelector):
    """A selector that randomly selects correct exemplars.

    This class implements a strategy that generates random examples and selects
    those that are evaluated as correct until the desired number of exemplars is reached.
    """

    def __init__(
        self, task: "BaseTask", predictor: "BasePredictor", desired_score: int = 1, config: "ExperimentConfig" = None
    ):
        """Initialize the RandomSelector.

        Args:
            task (BaseTask): An object representing the task to be performed.
            predictor (BasePredictor): An object capable of making predictions based on prompts.
            desired_score (int, optional): The desired score for the exemplars. Defaults to 1.
            config (ExperimentConfig, optional): Configuration for the selector, overriding defaults.
        """
        self.desired_score = desired_score
        super().__init__(task, predictor, config)

    def select_exemplars(self, prompt: str, n_examples: int = 5) -> str:
        """Select exemplars using a random selection strategy.

        This method generates random examples and selects those that are evaluated as correct
        (score == self.desired_score) until the desired number of exemplars is reached.

        Args:
            prompt (str): The input prompt to base the exemplar selection on.
            n_examples (int, optional): The number of exemplars to select. Defaults to 5.

        Returns:
            str: A new prompt that includes the original prompt and the selected exemplars.
        """
        examples = []
        while len(examples) < n_examples:
            score, seq = self.task.evaluate(prompt, self.predictor, n_samples=1, return_seq=True)
            if score == self.desired_score:
                examples.append(seq[0])
        prompt = "\n\n".join([prompt] + examples) + "\n\n"

        return prompt

__init__(task, predictor, desired_score=1, config=None)

Initialize the RandomSelector.

Parameters:

Name	Type	Description	Default
task	BaseTask	An object representing the task to be performed.	required
predictor	BasePredictor	An object capable of making predictions based on prompts.	required
desired_score	int	The desired score for the exemplars. Defaults to 1.	1
config	ExperimentConfig	Configuration for the selector, overriding defaults.	None

Source code in promptolution/exemplar_selectors/random_selector.py

def __init__(
    self, task: "BaseTask", predictor: "BasePredictor", desired_score: int = 1, config: "ExperimentConfig" = None
):
    """Initialize the RandomSelector.

    Args:
        task (BaseTask): An object representing the task to be performed.
        predictor (BasePredictor): An object capable of making predictions based on prompts.
        desired_score (int, optional): The desired score for the exemplars. Defaults to 1.
        config (ExperimentConfig, optional): Configuration for the selector, overriding defaults.
    """
    self.desired_score = desired_score
    super().__init__(task, predictor, config)

select_exemplars(prompt, n_examples=5)

Select exemplars using a random selection strategy.

This method generates random examples and selects those that are evaluated as correct (score == self.desired_score) until the desired number of exemplars is reached.

Parameters:

Name	Type	Description	Default
prompt	str	The input prompt to base the exemplar selection on.	required
n_examples	int	The number of exemplars to select. Defaults to 5.	5

Returns:

Name	Type	Description
str	str	A new prompt that includes the original prompt and the selected exemplars.

Source code in promptolution/exemplar_selectors/random_selector.py

def select_exemplars(self, prompt: str, n_examples: int = 5) -> str:
    """Select exemplars using a random selection strategy.

    This method generates random examples and selects those that are evaluated as correct
    (score == self.desired_score) until the desired number of exemplars is reached.

    Args:
        prompt (str): The input prompt to base the exemplar selection on.
        n_examples (int, optional): The number of exemplars to select. Defaults to 5.

    Returns:
        str: A new prompt that includes the original prompt and the selected exemplars.
    """
    examples = []
    while len(examples) < n_examples:
        score, seq = self.task.evaluate(prompt, self.predictor, n_samples=1, return_seq=True)
        if score == self.desired_score:
            examples.append(seq[0])
    prompt = "\n\n".join([prompt] + examples) + "\n\n"

    return prompt