Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 84 additions & 2 deletions src/Providers/ModelResolver.php
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,10 @@ public function resolve(
$candidateMap = $this->getCandidateModelsMap($requirements);

if (empty($candidateMap)) {
// An unsatisfiable option, rather than the requested capability, is a common cause of an
// empty candidate map. Naming it turns an opaque failure into an actionable one.
$optionSuffix = $this->describeUnsupportedOptions($requirements);

// The primary capability is always the first required capability (see
// ModelRequirements::fromPromptData()/fromEmbeddingData()).
$requiredCapabilities = $requirements->getRequiredCapabilities();
Expand All @@ -245,7 +249,7 @@ public function resolve(
);
}

throw new InvalidArgumentException($message);
throw new InvalidArgumentException($message . $optionSuffix);
}

$capabilityValue = $primaryCapability->value;
Expand Down Expand Up @@ -277,7 +281,7 @@ public function resolve(
}
}

throw new InvalidArgumentException($message);
throw new InvalidArgumentException($message . $optionSuffix);
}

// Check if any preferred models match the candidates, in priority order.
Expand Down Expand Up @@ -468,4 +472,82 @@ private function createModelPreferenceKey(string $modelId): string
{
return 'model::' . $modelId;
}

/**
* Describes which of the required options no otherwise-suitable model supports.
*
* When model resolution fails, the requested capability is often supported while a requested
* option is not. This method identifies the options that every model meeting the required
* capabilities fails to support, so that the resulting error can name the actual cause.
*
* @since n.e.x.t
*
* @param ModelRequirements $requirements The requirements that produced no candidates.
* @return string A sentence naming the unsupported options, or an empty string if the options
* are not the cause.
*/
private function describeUnsupportedOptions(ModelRequirements $requirements): string
{
if ($requirements->getRequiredOptions() === []) {
return '';
}

$modelsMetadata = $this->findMetadataMeetingCapabilities($requirements);
if ($modelsMetadata === []) {
// No model supports the capability either, so the capability is the cause.
return '';
}

$unsupported = null;
foreach ($modelsMetadata as $modelMetadata) {
$unmetNames = [];
foreach ($requirements->getUnmetRequirements($modelMetadata)['options'] as $unmetOption) {
$unmetNames[$unmetOption->getName()->value] = true;
}

$unsupported = $unsupported === null
? $unmetNames
: array_intersect_key($unsupported, $unmetNames);

if ($unsupported === []) {
// At least one model supports every requested option, so they are not the cause.
return '';
}
}

return sprintf(
' The following requested %s not supported by any of those models: %s.',
count($unsupported) === 1 ? 'option is' : 'options are',
implode(', ', array_keys($unsupported))
);
}

/**
* Finds the metadata of all models that meet the required capabilities, ignoring options.
*
* @since n.e.x.t
*
* @param ModelRequirements $requirements The requirements to take the capabilities from.
* @return list<ModelMetadata> The metadata of the models meeting the required capabilities.
*/
private function findMetadataMeetingCapabilities(ModelRequirements $requirements): array
{
$capabilityOnlyRequirements = new ModelRequirements($requirements->getRequiredCapabilities(), []);

if ($this->providerIdOrClassName !== null) {
return $this->registry->findProviderModelsMetadataForSupport(
$this->providerIdOrClassName,
$capabilityOnlyRequirements
);
}

$modelsMetadata = [];
foreach ($this->registry->findModelsMetadataForSupport($capabilityOnlyRequirements) as $providerModels) {
foreach ($providerModels->getModels() as $modelMetadata) {
$modelsMetadata[] = $modelMetadata;
}
}

return $modelsMetadata;
}
}
88 changes: 82 additions & 6 deletions src/Tools/DTO/WebSearch.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,32 @@

namespace WordPress\AiClient\Tools\DTO;

use InvalidArgumentException;
use WordPress\AiClient\Common\AbstractDataTransferObject;

/**
* Represents web search configuration for AI models.
*
* This DTO defines constraints for web searches that AI models can perform,
* including allowed and disallowed domains.
* including allowed and disallowed domains, and a bag of provider specific
* options for settings that have no portable equivalent.
*
* @since 0.1.0
*
* @phpstan-type WebSearchArrayShape array{allowedDomains?: string[], disallowedDomains?: string[]}
* @phpstan-type WebSearchArrayShape array{
* allowedDomains?: string[],
* disallowedDomains?: string[],
* providerOptions?: array<string, array<string, mixed>>
* }
*
* @extends AbstractDataTransferObject<WebSearchArrayShape>
*/
class WebSearch extends AbstractDataTransferObject
{
public const KEY_ALLOWED_DOMAINS = 'allowedDomains';
public const KEY_DISALLOWED_DOMAINS = 'disallowedDomains';
public const KEY_PROVIDER_OPTIONS = 'providerOptions';

/**
* @var string[] List of domains that are allowed for web search.
*/
Expand All @@ -32,18 +40,47 @@ class WebSearch extends AbstractDataTransferObject
*/
private array $disallowedDomains;

/**
* @var array<string, array<string, mixed>> Provider specific options, keyed by provider ID.
*/
private array $providerOptions;

/**
* Constructor.
*
* @since 0.1.0
* @since n.e.x.t Adds the optional $providerOptions parameter.
*
* @param string[] $allowedDomains List of domains that are allowed for web search.
* @param string[] $disallowedDomains List of domains that are disallowed for web search.
* @param array<string, array<string, mixed>> $providerOptions Provider specific web search options, keyed by
* provider ID. Only the entry matching the resolved
* provider is used.
* @throws InvalidArgumentException If the provider options are not keyed by provider ID, or an entry is not
* an array.
*/
public function __construct(array $allowedDomains = [], array $disallowedDomains = [])
{
public function __construct(
array $allowedDomains = [],
array $disallowedDomains = [],
array $providerOptions = []
) {
foreach ($providerOptions as $providerId => $options) {
if (!is_string($providerId) || $providerId === '') {
throw new InvalidArgumentException(
'Web search provider options must be keyed by a non-empty provider ID.'
);
}

if (!is_array($options)) {
throw new InvalidArgumentException(
sprintf('Web search provider options for "%s" must be an array.', $providerId)
);
}
}

$this->allowedDomains = $allowedDomains;
$this->disallowedDomains = $disallowedDomains;
$this->providerOptions = $providerOptions;
}

/**
Expand All @@ -70,6 +107,31 @@ public function getDisallowedDomains(): array
return $this->disallowedDomains;
}

/**
* Gets the provider specific options for all providers.
*
* @since n.e.x.t
*
* @return array<string, array<string, mixed>> The provider specific options, keyed by provider ID.
*/
public function getProviderOptions(): array
{
return $this->providerOptions;
}

/**
* Gets the provider specific options for a single provider.
*
* @since n.e.x.t
*
* @param string $providerId The provider ID to get the options for.
* @return array<string, mixed> The options for the provider, or an empty array if none were provided.
*/
public function getProviderOptionsFor(string $providerId): array
{
return $this->providerOptions[$providerId] ?? [];
}

/**
* {@inheritDoc}
*
Expand All @@ -94,6 +156,13 @@ public static function getJsonSchema(): array
],
'description' => 'List of domains that are disallowed for web search.',
],
self::KEY_PROVIDER_OPTIONS => [
'type' => 'object',
'additionalProperties' => [
'type' => 'object',
],
'description' => 'Provider specific web search options, keyed by provider ID.',
],
],
'required' => [],
];
Expand All @@ -108,10 +177,16 @@ public static function getJsonSchema(): array
*/
public function toArray(): array
{
return [
$data = [
self::KEY_ALLOWED_DOMAINS => $this->allowedDomains,
self::KEY_DISALLOWED_DOMAINS => $this->disallowedDomains,
];

if ($this->providerOptions !== []) {
$data[self::KEY_PROVIDER_OPTIONS] = $this->providerOptions;
}

return $data;
}

/**
Expand All @@ -123,7 +198,8 @@ public static function fromArray(array $array): self
{
return new self(
$array[self::KEY_ALLOWED_DOMAINS] ?? [],
$array[self::KEY_DISALLOWED_DOMAINS] ?? []
$array[self::KEY_DISALLOWED_DOMAINS] ?? [],
$array[self::KEY_PROVIDER_OPTIONS] ?? []
);
}
}
6 changes: 4 additions & 2 deletions tests/unit/Builders/PromptBuilderTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -3513,8 +3513,10 @@ public function testGenerateResultWithProviderClassName(): void
*/
public function testGenerateResultWithProviderNoModelsThrowsException(): void
{
// Mock the registry to return empty array when provider is specified
$this->registry->expects($this->once())
// Mock the registry to return empty array when provider is specified. The resolver looks
// twice on failure: once with the required options applied, then once without them to work
// out whether an option rather than the capability is to blame.
$this->registry->expects($this->exactly(2))
->method('findProviderModelsMetadataForSupport')
->with('test-provider', $this->isInstanceOf(ModelRequirements::class))
->willReturn([]);
Expand Down
97 changes: 97 additions & 0 deletions tests/unit/Providers/ModelResolverTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
use WordPress\AiClient\Providers\Models\Contracts\ModelInterface;
use WordPress\AiClient\Providers\Models\DTO\ModelConfig;
use WordPress\AiClient\Providers\Models\DTO\ModelRequirements;
use WordPress\AiClient\Providers\Models\DTO\RequiredOption;
use WordPress\AiClient\Providers\Models\Enums\CapabilityEnum;
use WordPress\AiClient\Providers\Models\Enums\OptionEnum;
use WordPress\AiClient\Providers\ProviderRegistry;
use WordPress\AiClient\Tests\traits\MockModelCreationTrait;

Expand Down Expand Up @@ -467,4 +469,99 @@ public function testCloneWorksWithNullRequestOptions(): void

$this->assertNull($this->getResolverProperty($cloned, 'requestOptions'));
}

/**
* Tests resolve names the unsatisfied option when the capability itself is supported.
*
* @return void
*/
public function testResolveNamesUnsupportedOptionWhenCapabilityIsSupported(): void
{
$metadata = $this->createTestTextModelMetadata();
$providerMetadata = new ProviderMetadata('mock', 'Mock Provider', ProviderTypeEnum::cloud());

// The first lookup applies the options and finds nothing; the second drops them and
// finds a model, proving the option rather than the capability is the cause.
$this->registry->expects($this->exactly(2))
->method('findModelsMetadataForSupport')
->willReturnOnConsecutiveCalls(
[],
[new ProviderModelsMetadata($providerMetadata, [$metadata])]
);

$requirements = new ModelRequirements(
[CapabilityEnum::textGeneration()],
[new RequiredOption(OptionEnum::webSearch(), true)]
);

$resolver = new ModelResolver($this->registry);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage(
'No models found that support text_generation.'
. ' The following requested option is not supported by any of those models: webSearch.'
);

$resolver->resolve($requirements, new ModelConfig());
}

/**
* Tests resolve does not blame an option when no model supports the capability either.
*
* @return void
*/
public function testResolveOmitsOptionDetailWhenCapabilityIsUnsupported(): void
{
$this->registry->expects($this->exactly(2))
->method('findModelsMetadataForSupport')
->willReturn([]);

$requirements = new ModelRequirements(
[CapabilityEnum::textGeneration()],
[new RequiredOption(OptionEnum::webSearch(), true)]
);

$resolver = new ModelResolver($this->registry);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage('No models found that support text_generation.');

$resolver->resolve($requirements, new ModelConfig());
}

/**
* Tests resolve names every option that no otherwise-suitable model supports.
*
* @return void
*/
public function testResolveNamesAllUnsupportedOptions(): void
{
$metadata = $this->createTestTextModelMetadata();
$providerMetadata = new ProviderMetadata('mock', 'Mock Provider', ProviderTypeEnum::cloud());

$this->registry->expects($this->exactly(2))
->method('findModelsMetadataForSupport')
->willReturnOnConsecutiveCalls(
[],
[new ProviderModelsMetadata($providerMetadata, [$metadata])]
);

$requirements = new ModelRequirements(
[CapabilityEnum::textGeneration()],
[
new RequiredOption(OptionEnum::webSearch(), true),
new RequiredOption(OptionEnum::functionDeclarations(), true),
]
);

$resolver = new ModelResolver($this->registry);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage(
'The following requested options are not supported by any of those models: '
. 'webSearch, functionDeclarations.'
);

$resolver->resolve($requirements, new ModelConfig());
}
}
Loading
Loading