diff --git a/src/Providers/ModelResolver.php b/src/Providers/ModelResolver.php index 33a7557e..cd7d71ff 100644 --- a/src/Providers/ModelResolver.php +++ b/src/Providers/ModelResolver.php @@ -231,6 +231,10 @@ public function resolve( $candidateMap = $this->getCandidateModelsMap($requirements); if (empty($candidateMap)) { + // An unsatisfiable option, rather than the requested capability, is a common cause of an + // empty candidate map. Naming it turns an opaque failure into an actionable one. + $optionSuffix = $this->describeUnsupportedOptions($requirements); + // The primary capability is always the first required capability (see // ModelRequirements::fromPromptData()/fromEmbeddingData()). $requiredCapabilities = $requirements->getRequiredCapabilities(); @@ -245,7 +249,7 @@ public function resolve( ); } - throw new InvalidArgumentException($message); + throw new InvalidArgumentException($message . $optionSuffix); } $capabilityValue = $primaryCapability->value; @@ -277,7 +281,7 @@ public function resolve( } } - throw new InvalidArgumentException($message); + throw new InvalidArgumentException($message . $optionSuffix); } // Check if any preferred models match the candidates, in priority order. @@ -468,4 +472,82 @@ private function createModelPreferenceKey(string $modelId): string { return 'model::' . $modelId; } + + /** + * Describes which of the required options no otherwise-suitable model supports. + * + * When model resolution fails, the requested capability is often supported while a requested + * option is not. This method identifies the options that every model meeting the required + * capabilities fails to support, so that the resulting error can name the actual cause. + * + * @since n.e.x.t + * + * @param ModelRequirements $requirements The requirements that produced no candidates. + * @return string A sentence naming the unsupported options, or an empty string if the options + * are not the cause. + */ + private function describeUnsupportedOptions(ModelRequirements $requirements): string + { + if ($requirements->getRequiredOptions() === []) { + return ''; + } + + $modelsMetadata = $this->findMetadataMeetingCapabilities($requirements); + if ($modelsMetadata === []) { + // No model supports the capability either, so the capability is the cause. + return ''; + } + + $unsupported = null; + foreach ($modelsMetadata as $modelMetadata) { + $unmetNames = []; + foreach ($requirements->getUnmetRequirements($modelMetadata)['options'] as $unmetOption) { + $unmetNames[$unmetOption->getName()->value] = true; + } + + $unsupported = $unsupported === null + ? $unmetNames + : array_intersect_key($unsupported, $unmetNames); + + if ($unsupported === []) { + // At least one model supports every requested option, so they are not the cause. + return ''; + } + } + + return sprintf( + ' The following requested %s not supported by any of those models: %s.', + count($unsupported) === 1 ? 'option is' : 'options are', + implode(', ', array_keys($unsupported)) + ); + } + + /** + * Finds the metadata of all models that meet the required capabilities, ignoring options. + * + * @since n.e.x.t + * + * @param ModelRequirements $requirements The requirements to take the capabilities from. + * @return list The metadata of the models meeting the required capabilities. + */ + private function findMetadataMeetingCapabilities(ModelRequirements $requirements): array + { + $capabilityOnlyRequirements = new ModelRequirements($requirements->getRequiredCapabilities(), []); + + if ($this->providerIdOrClassName !== null) { + return $this->registry->findProviderModelsMetadataForSupport( + $this->providerIdOrClassName, + $capabilityOnlyRequirements + ); + } + + $modelsMetadata = []; + foreach ($this->registry->findModelsMetadataForSupport($capabilityOnlyRequirements) as $providerModels) { + foreach ($providerModels->getModels() as $modelMetadata) { + $modelsMetadata[] = $modelMetadata; + } + } + + return $modelsMetadata; + } } diff --git a/src/Tools/DTO/WebSearch.php b/src/Tools/DTO/WebSearch.php index 212cf48d..cb0d2626 100644 --- a/src/Tools/DTO/WebSearch.php +++ b/src/Tools/DTO/WebSearch.php @@ -4,17 +4,23 @@ namespace WordPress\AiClient\Tools\DTO; +use InvalidArgumentException; use WordPress\AiClient\Common\AbstractDataTransferObject; /** * Represents web search configuration for AI models. * * This DTO defines constraints for web searches that AI models can perform, - * including allowed and disallowed domains. + * including allowed and disallowed domains, and a bag of provider specific + * options for settings that have no portable equivalent. * * @since 0.1.0 * - * @phpstan-type WebSearchArrayShape array{allowedDomains?: string[], disallowedDomains?: string[]} + * @phpstan-type WebSearchArrayShape array{ + * allowedDomains?: string[], + * disallowedDomains?: string[], + * providerOptions?: array> + * } * * @extends AbstractDataTransferObject */ @@ -22,6 +28,8 @@ class WebSearch extends AbstractDataTransferObject { public const KEY_ALLOWED_DOMAINS = 'allowedDomains'; public const KEY_DISALLOWED_DOMAINS = 'disallowedDomains'; + public const KEY_PROVIDER_OPTIONS = 'providerOptions'; + /** * @var string[] List of domains that are allowed for web search. */ @@ -32,18 +40,47 @@ class WebSearch extends AbstractDataTransferObject */ private array $disallowedDomains; + /** + * @var array> Provider specific options, keyed by provider ID. + */ + private array $providerOptions; + /** * Constructor. * * @since 0.1.0 + * @since n.e.x.t Adds the optional $providerOptions parameter. * * @param string[] $allowedDomains List of domains that are allowed for web search. * @param string[] $disallowedDomains List of domains that are disallowed for web search. + * @param array> $providerOptions Provider specific web search options, keyed by + * provider ID. Only the entry matching the resolved + * provider is used. + * @throws InvalidArgumentException If the provider options are not keyed by provider ID, or an entry is not + * an array. */ - public function __construct(array $allowedDomains = [], array $disallowedDomains = []) - { + public function __construct( + array $allowedDomains = [], + array $disallowedDomains = [], + array $providerOptions = [] + ) { + foreach ($providerOptions as $providerId => $options) { + if (!is_string($providerId) || $providerId === '') { + throw new InvalidArgumentException( + 'Web search provider options must be keyed by a non-empty provider ID.' + ); + } + + if (!is_array($options)) { + throw new InvalidArgumentException( + sprintf('Web search provider options for "%s" must be an array.', $providerId) + ); + } + } + $this->allowedDomains = $allowedDomains; $this->disallowedDomains = $disallowedDomains; + $this->providerOptions = $providerOptions; } /** @@ -70,6 +107,31 @@ public function getDisallowedDomains(): array return $this->disallowedDomains; } + /** + * Gets the provider specific options for all providers. + * + * @since n.e.x.t + * + * @return array> The provider specific options, keyed by provider ID. + */ + public function getProviderOptions(): array + { + return $this->providerOptions; + } + + /** + * Gets the provider specific options for a single provider. + * + * @since n.e.x.t + * + * @param string $providerId The provider ID to get the options for. + * @return array The options for the provider, or an empty array if none were provided. + */ + public function getProviderOptionsFor(string $providerId): array + { + return $this->providerOptions[$providerId] ?? []; + } + /** * {@inheritDoc} * @@ -94,6 +156,13 @@ public static function getJsonSchema(): array ], 'description' => 'List of domains that are disallowed for web search.', ], + self::KEY_PROVIDER_OPTIONS => [ + 'type' => 'object', + 'additionalProperties' => [ + 'type' => 'object', + ], + 'description' => 'Provider specific web search options, keyed by provider ID.', + ], ], 'required' => [], ]; @@ -108,10 +177,16 @@ public static function getJsonSchema(): array */ public function toArray(): array { - return [ + $data = [ self::KEY_ALLOWED_DOMAINS => $this->allowedDomains, self::KEY_DISALLOWED_DOMAINS => $this->disallowedDomains, ]; + + if ($this->providerOptions !== []) { + $data[self::KEY_PROVIDER_OPTIONS] = $this->providerOptions; + } + + return $data; } /** @@ -123,7 +198,8 @@ public static function fromArray(array $array): self { return new self( $array[self::KEY_ALLOWED_DOMAINS] ?? [], - $array[self::KEY_DISALLOWED_DOMAINS] ?? [] + $array[self::KEY_DISALLOWED_DOMAINS] ?? [], + $array[self::KEY_PROVIDER_OPTIONS] ?? [] ); } } diff --git a/tests/unit/Builders/PromptBuilderTest.php b/tests/unit/Builders/PromptBuilderTest.php index 23a58959..211fddd6 100644 --- a/tests/unit/Builders/PromptBuilderTest.php +++ b/tests/unit/Builders/PromptBuilderTest.php @@ -3513,8 +3513,10 @@ public function testGenerateResultWithProviderClassName(): void */ public function testGenerateResultWithProviderNoModelsThrowsException(): void { - // Mock the registry to return empty array when provider is specified - $this->registry->expects($this->once()) + // Mock the registry to return empty array when provider is specified. The resolver looks + // twice on failure: once with the required options applied, then once without them to work + // out whether an option rather than the capability is to blame. + $this->registry->expects($this->exactly(2)) ->method('findProviderModelsMetadataForSupport') ->with('test-provider', $this->isInstanceOf(ModelRequirements::class)) ->willReturn([]); diff --git a/tests/unit/Providers/ModelResolverTest.php b/tests/unit/Providers/ModelResolverTest.php index 835840cd..13382c28 100644 --- a/tests/unit/Providers/ModelResolverTest.php +++ b/tests/unit/Providers/ModelResolverTest.php @@ -15,7 +15,9 @@ use WordPress\AiClient\Providers\Models\Contracts\ModelInterface; use WordPress\AiClient\Providers\Models\DTO\ModelConfig; use WordPress\AiClient\Providers\Models\DTO\ModelRequirements; +use WordPress\AiClient\Providers\Models\DTO\RequiredOption; use WordPress\AiClient\Providers\Models\Enums\CapabilityEnum; +use WordPress\AiClient\Providers\Models\Enums\OptionEnum; use WordPress\AiClient\Providers\ProviderRegistry; use WordPress\AiClient\Tests\traits\MockModelCreationTrait; @@ -467,4 +469,99 @@ public function testCloneWorksWithNullRequestOptions(): void $this->assertNull($this->getResolverProperty($cloned, 'requestOptions')); } + + /** + * Tests resolve names the unsatisfied option when the capability itself is supported. + * + * @return void + */ + public function testResolveNamesUnsupportedOptionWhenCapabilityIsSupported(): void + { + $metadata = $this->createTestTextModelMetadata(); + $providerMetadata = new ProviderMetadata('mock', 'Mock Provider', ProviderTypeEnum::cloud()); + + // The first lookup applies the options and finds nothing; the second drops them and + // finds a model, proving the option rather than the capability is the cause. + $this->registry->expects($this->exactly(2)) + ->method('findModelsMetadataForSupport') + ->willReturnOnConsecutiveCalls( + [], + [new ProviderModelsMetadata($providerMetadata, [$metadata])] + ); + + $requirements = new ModelRequirements( + [CapabilityEnum::textGeneration()], + [new RequiredOption(OptionEnum::webSearch(), true)] + ); + + $resolver = new ModelResolver($this->registry); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'No models found that support text_generation.' + . ' The following requested option is not supported by any of those models: webSearch.' + ); + + $resolver->resolve($requirements, new ModelConfig()); + } + + /** + * Tests resolve does not blame an option when no model supports the capability either. + * + * @return void + */ + public function testResolveOmitsOptionDetailWhenCapabilityIsUnsupported(): void + { + $this->registry->expects($this->exactly(2)) + ->method('findModelsMetadataForSupport') + ->willReturn([]); + + $requirements = new ModelRequirements( + [CapabilityEnum::textGeneration()], + [new RequiredOption(OptionEnum::webSearch(), true)] + ); + + $resolver = new ModelResolver($this->registry); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('No models found that support text_generation.'); + + $resolver->resolve($requirements, new ModelConfig()); + } + + /** + * Tests resolve names every option that no otherwise-suitable model supports. + * + * @return void + */ + public function testResolveNamesAllUnsupportedOptions(): void + { + $metadata = $this->createTestTextModelMetadata(); + $providerMetadata = new ProviderMetadata('mock', 'Mock Provider', ProviderTypeEnum::cloud()); + + $this->registry->expects($this->exactly(2)) + ->method('findModelsMetadataForSupport') + ->willReturnOnConsecutiveCalls( + [], + [new ProviderModelsMetadata($providerMetadata, [$metadata])] + ); + + $requirements = new ModelRequirements( + [CapabilityEnum::textGeneration()], + [ + new RequiredOption(OptionEnum::webSearch(), true), + new RequiredOption(OptionEnum::functionDeclarations(), true), + ] + ); + + $resolver = new ModelResolver($this->registry); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'The following requested options are not supported by any of those models: ' + . 'webSearch, functionDeclarations.' + ); + + $resolver->resolve($requirements, new ModelConfig()); + } } diff --git a/tests/unit/Tools/DTO/WebSearchTest.php b/tests/unit/Tools/DTO/WebSearchTest.php index 465478c9..bd866b4c 100644 --- a/tests/unit/Tools/DTO/WebSearchTest.php +++ b/tests/unit/Tools/DTO/WebSearchTest.php @@ -4,6 +4,7 @@ namespace WordPress\AiClient\Tests\unit\Tools\DTO; +use InvalidArgumentException; use PHPUnit\Framework\TestCase; use WordPress\AiClient\Common\Contracts\WithJsonSchemaInterface; use WordPress\AiClient\Tests\traits\ArrayTransformationTestTrait; @@ -449,4 +450,190 @@ public function testImplementsWithArrayTransformationInterface(): void $webSearch = new WebSearch(); $this->assertImplementsArrayTransformation($webSearch); } + + /** + * Tests creating WebSearch with provider specific options. + * + * @return void + */ + public function testCreateWithProviderOptions(): void + { + $webSearch = new WebSearch(['example.com'], [], ['anthropic' => ['max_uses' => 3]]); + + $this->assertSame(['anthropic' => ['max_uses' => 3]], $webSearch->getProviderOptions()); + } + + /** + * Tests the provider options default to an empty array. + * + * @return void + */ + public function testProviderOptionsDefaultToEmptyArray(): void + { + $webSearch = new WebSearch(); + + $this->assertSame([], $webSearch->getProviderOptions()); + } + + /** + * Tests reading the provider options for a single provider. + * + * @return void + */ + public function testGetProviderOptionsForReturnsOptionsOfThatProviderOnly(): void + { + $webSearch = new WebSearch([], [], [ + 'anthropic' => ['max_uses' => 3], + 'openai' => ['search_context_size' => 'low'], + ]); + + $this->assertSame(['max_uses' => 3], $webSearch->getProviderOptionsFor('anthropic')); + $this->assertSame(['search_context_size' => 'low'], $webSearch->getProviderOptionsFor('openai')); + } + + /** + * Tests reading the provider options for a provider without any returns an empty array. + * + * @return void + */ + public function testGetProviderOptionsForUnknownProviderReturnsEmptyArray(): void + { + $webSearch = new WebSearch([], [], ['anthropic' => ['max_uses' => 3]]); + + $this->assertSame([], $webSearch->getProviderOptionsFor('google')); + } + + /** + * Tests provider options keyed by something other than a provider ID are rejected. + * + * @return void + */ + public function testRejectsProviderOptionsWithNonStringKey(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('must be keyed by a non-empty provider ID'); + + new WebSearch([], [], [['max_uses' => 3]]); + } + + /** + * Tests provider options keyed by an empty provider ID are rejected. + * + * @return void + */ + public function testRejectsProviderOptionsWithEmptyProviderId(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('must be keyed by a non-empty provider ID'); + + new WebSearch([], [], ['' => ['max_uses' => 3]]); + } + + /** + * Tests a provider options entry that is not an array is rejected. + * + * @return void + */ + public function testRejectsNonArrayProviderOptionsEntry(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Web search provider options for "anthropic" must be an array.'); + + /** @phpstan-ignore-next-line Intentionally invalid to assert the guard. */ + new WebSearch([], [], ['anthropic' => 'max_uses=3']); + } + + /** + * Tests array transformation omits the provider options when none are set. + * + * @return void + */ + public function testToArrayOmitsProviderOptionsWhenEmpty(): void + { + $json = $this->assertToArrayReturnsArray(new WebSearch(['example.com'])); + + $this->assertArrayNotHasKey(WebSearch::KEY_PROVIDER_OPTIONS, $json); + } + + /** + * Tests array transformation includes the provider options when they are set. + * + * @return void + */ + public function testToArrayIncludesProviderOptions(): void + { + $json = $this->assertToArrayReturnsArray( + new WebSearch(['example.com'], [], ['anthropic' => ['max_uses' => 5]]) + ); + + $this->assertArrayHasKey(WebSearch::KEY_PROVIDER_OPTIONS, $json); + $this->assertSame(['anthropic' => ['max_uses' => 5]], $json[WebSearch::KEY_PROVIDER_OPTIONS]); + } + + /** + * Tests fromArray reads the provider options. + * + * @return void + */ + public function testFromArrayWithProviderOptions(): void + { + $webSearch = WebSearch::fromArray([ + WebSearch::KEY_ALLOWED_DOMAINS => ['example.com'], + WebSearch::KEY_PROVIDER_OPTIONS => ['anthropic' => ['max_uses' => 2]], + ]); + + $this->assertSame(['example.com'], $webSearch->getAllowedDomains()); + $this->assertSame(['max_uses' => 2], $webSearch->getProviderOptionsFor('anthropic')); + } + + /** + * Tests fromArray defaults the provider options to an empty array when absent. + * + * @return void + */ + public function testFromArrayWithoutProviderOptionsDefaultsToEmptyArray(): void + { + $webSearch = WebSearch::fromArray([ + WebSearch::KEY_ALLOWED_DOMAINS => ['example.com'], + ]); + + $this->assertSame([], $webSearch->getProviderOptions()); + } + + /** + * Tests round-trip array transformation preserves the provider options. + * + * @return void + */ + public function testArrayRoundTripWithProviderOptions(): void + { + $this->assertArrayRoundTrip( + new WebSearch( + ['wikipedia.org'], + [], + ['anthropic' => ['max_uses' => 4, 'user_location' => ['type' => 'approximate', 'country' => 'FI']]] + ), + function ($original, $restored) { + $this->assertEquals($original->getProviderOptions(), $restored->getProviderOptions()); + $this->assertEquals($original->getAllowedDomains(), $restored->getAllowedDomains()); + } + ); + } + + /** + * Tests the JSON schema describes the provider options. + * + * @return void + */ + public function testJsonSchemaDescribesProviderOptions(): void + { + $schema = WebSearch::getJsonSchema(); + + $this->assertArrayHasKey(WebSearch::KEY_PROVIDER_OPTIONS, $schema['properties']); + + $providerOptionsSchema = $schema['properties'][WebSearch::KEY_PROVIDER_OPTIONS]; + $this->assertEquals('object', $providerOptionsSchema['type']); + $this->assertEquals(['type' => 'object'], $providerOptionsSchema['additionalProperties']); + $this->assertArrayHasKey('description', $providerOptionsSchema); + } }