From 06ed5c509559da62bac375b939c345d9fc317ac2 Mon Sep 17 00:00:00 2001 From: nelsw Date: Mon, 28 Sep 2026 14:55:20 -0400 Subject: [PATCH 1/2] removed remote logic to local engine --- app/Actions/Model/UpdateArticle.php | 32 --- app/Actions/Model/UpdateBotRunTimestamp.php | 15 -- .../Model/UpdateOrCreateSitemapPage.php | 51 ---- app/Actions/Model/UpdateSitemapUrls.php | 25 -- app/Concerns/PageValidationRules.php | 19 -- app/Concerns/ScrapeValidationRules.php | 17 -- app/Concerns/SerpValidationRules.php | 18 -- app/Concerns/SitemapPageValidationRules.php | 23 -- app/Concerns/SitemapValidationRules.php | 17 -- app/Console/Commands/GoPub.php | 20 ++ app/Console/Commands/GoSub.php | 20 ++ app/Enums/NewsSource.php | 9 - app/Facades/Go.php | 20 ++ app/Facades/Rss.php | 18 -- app/Facades/S3.php | 18 ++ app/Facades/Sqs.php | 19 -- .../Controllers/Api/ScrapeJobController.php | 110 -------- app/Http/Requests/Api/PageSaveRequest.php | 17 -- app/Jobs/BotJob.php | 82 +----- app/Models/Article.php | 2 +- app/Models/Bot.php | 9 + app/Observers/SerpObserver.php | 4 +- app/Providers/MacroServiceProvider.php | 51 +--- app/Services/GoService.php | 61 +++++ app/Services/RssService.php | 54 ---- app/Services/S3Service.php | 44 +++ app/Services/SqsService.php | 43 --- app/Support/Rss/BingRssItem.php | 29 -- app/Support/Rss/GoogleRssItem.php | 183 ------------- app/Support/Rss/RssItem.php | 57 ---- app/Support/Sqs/Html.php | 63 ----- app/Support/Sqs/Page.php | 42 --- app/Support/Sqs/Serp.php | 191 ------------- app/Traits/HasBody.php | 27 -- app/Traits/HasLinks.php | 66 ----- app/Traits/HasMeta.php | 93 ------- app/Traits/HasNewsSource.php | 20 -- app/Traits/HasScreenshot.php | 19 +- app/Traits/Services/HasSqsClient.php | 5 - compose.yml | 2 +- config/database.php | 14 + routes/api.php | 9 +- tests/Unit/Support/Rss/BaseRssItemTest.php | 87 ------ tests/Unit/Support/Rss/BingRssItemTest.php | 71 ----- tests/Unit/Support/Rss/GoogleRssItemTest.php | 251 ------------------ tests/Unit/Support/Sqs/PageTest.php | 156 ----------- 46 files changed, 223 insertions(+), 1980 deletions(-) delete mode 100644 app/Actions/Model/UpdateArticle.php delete mode 100644 app/Actions/Model/UpdateBotRunTimestamp.php delete mode 100644 app/Actions/Model/UpdateOrCreateSitemapPage.php delete mode 100644 app/Actions/Model/UpdateSitemapUrls.php delete mode 100644 app/Concerns/PageValidationRules.php delete mode 100644 app/Concerns/ScrapeValidationRules.php delete mode 100644 app/Concerns/SerpValidationRules.php delete mode 100644 app/Concerns/SitemapPageValidationRules.php delete mode 100644 app/Concerns/SitemapValidationRules.php create mode 100644 app/Console/Commands/GoPub.php create mode 100644 app/Console/Commands/GoSub.php delete mode 100644 app/Enums/NewsSource.php create mode 100644 app/Facades/Go.php delete mode 100644 app/Facades/Rss.php create mode 100644 app/Facades/S3.php delete mode 100644 app/Facades/Sqs.php delete mode 100644 app/Http/Controllers/Api/ScrapeJobController.php delete mode 100644 app/Http/Requests/Api/PageSaveRequest.php create mode 100644 app/Services/GoService.php delete mode 100644 app/Services/RssService.php create mode 100644 app/Services/S3Service.php delete mode 100644 app/Services/SqsService.php delete mode 100644 app/Support/Rss/BingRssItem.php delete mode 100644 app/Support/Rss/GoogleRssItem.php delete mode 100644 app/Support/Rss/RssItem.php delete mode 100644 app/Support/Sqs/Html.php delete mode 100644 app/Support/Sqs/Page.php delete mode 100644 app/Support/Sqs/Serp.php delete mode 100644 app/Traits/HasBody.php delete mode 100644 app/Traits/HasLinks.php delete mode 100644 app/Traits/HasMeta.php delete mode 100644 app/Traits/HasNewsSource.php delete mode 100644 app/Traits/Services/HasSqsClient.php delete mode 100644 tests/Unit/Support/Rss/BaseRssItemTest.php delete mode 100644 tests/Unit/Support/Rss/BingRssItemTest.php delete mode 100644 tests/Unit/Support/Rss/GoogleRssItemTest.php delete mode 100644 tests/Unit/Support/Sqs/PageTest.php diff --git a/app/Actions/Model/UpdateArticle.php b/app/Actions/Model/UpdateArticle.php deleted file mode 100644 index 3a09280..0000000 --- a/app/Actions/Model/UpdateArticle.php +++ /dev/null @@ -1,32 +0,0 @@ - $keywords */ - public function __invoke( - Article $article, - string $body, - string $description, - string $imgAlt, - string $imgUrl, - array $keywords, - ): void { - - when(empty($article->description), fn () => $article->setAttribute('description', $description)); - when(empty($article->img_alt), fn () => $article->setAttribute('img_alt', $imgAlt)); - when(empty($article->img_url), fn () => $article->setAttribute('img_url', $imgUrl)); - $article->setAttribute('body', $body); - $article->setAttribute('keywords', $keywords); - $article->save(); - - Log::debug('Article updated', $article->toArray()); - } -} diff --git a/app/Actions/Model/UpdateBotRunTimestamp.php b/app/Actions/Model/UpdateBotRunTimestamp.php deleted file mode 100644 index 07a27ca..0000000 --- a/app/Actions/Model/UpdateBotRunTimestamp.php +++ /dev/null @@ -1,15 +0,0 @@ -update(['last_run_at' => now()->utc()]); - Log::debug('Bot last_run_at updated'); - } -} diff --git a/app/Actions/Model/UpdateOrCreateSitemapPage.php b/app/Actions/Model/UpdateOrCreateSitemapPage.php deleted file mode 100644 index 205338c..0000000 --- a/app/Actions/Model/UpdateOrCreateSitemapPage.php +++ /dev/null @@ -1,51 +0,0 @@ - $bot->sitemap?->domain, - 'meta' => $meta, - 'pageable_id' => $bot->sitemap?->id, - 'pageable_type' => Sitemap::class, - 'screenshot_key' => $screenshotKey, - 'title' => $title, - 'url' => $url, - ], $this->sitemapPageRules()); - - /** @var Page $page */ - $page = $bot->sitemap?->pages()->updateOrCreate([ - 'url' => $url, - 'pageable_type' => Sitemap::class, - 'pageable_id' => $bot->sitemap?->id, - ], $values); - - Log::debug('Page '.($page->wasRecentlyCreated ? 'created' : 'updated'), [ - 'id' => $page->id, - 'sitemap' => $bot->sitemap?->id, - 'domain' => $bot->sitemap?->domain, - 'title' => $page->title, - 'url' => $page->url, - ]); - } -} diff --git a/app/Actions/Model/UpdateSitemapUrls.php b/app/Actions/Model/UpdateSitemapUrls.php deleted file mode 100644 index 4f13738..0000000 --- a/app/Actions/Model/UpdateSitemapUrls.php +++ /dev/null @@ -1,25 +0,0 @@ -urls; - ksort($urls); - $sitemap->urls = $urls; - - $sitemap->save(); - - Log::debug('Sitemap saved', [ - 'bot' => $sitemap->bot_id, - 'id' => $sitemap->id, - 'domain' => $sitemap->domain, - 'urls' => count($sitemap->urls), - ]); - } -} diff --git a/app/Concerns/PageValidationRules.php b/app/Concerns/PageValidationRules.php deleted file mode 100644 index c355839..0000000 --- a/app/Concerns/PageValidationRules.php +++ /dev/null @@ -1,19 +0,0 @@ - */ - protected function pageRules(): array - { - return [ - 'domain' => ['nullable', 'string', 'max:255'], - 'meta' => ['nullable', 'array'], - 'title' => ['required', 'string', 'max:1025'], - 'url' => ['required', 'string', 'url'], - 'index' => ['nullable', 'integer'], - 'kind' => ['nullable', 'string', 'max:255'], - ]; - } -} diff --git a/app/Concerns/ScrapeValidationRules.php b/app/Concerns/ScrapeValidationRules.php deleted file mode 100644 index fada627..0000000 --- a/app/Concerns/ScrapeValidationRules.php +++ /dev/null @@ -1,17 +0,0 @@ - */ - protected function scrapeRules(): array - { - return [ - 'screenshot_key' => ['required', 'string', 'max:1024'], - 'content_key' => ['required', 'string', 'max:1024'], - 'url' => ['required', 'url', 'max:2048'], - 'depth' => ['nullable', 'integer', 'min:0'], - ]; - } -} diff --git a/app/Concerns/SerpValidationRules.php b/app/Concerns/SerpValidationRules.php deleted file mode 100644 index 051f2ae..0000000 --- a/app/Concerns/SerpValidationRules.php +++ /dev/null @@ -1,18 +0,0 @@ - */ - protected function serpRules(): array - { - return [ - 'bot_id' => ['sometimes', 'required', 'integer', 'exists:bots,id'], - 'content_key' => ['nullable', 'string', 'max:255'], - 'data' => ['nullable', 'array'], - 'query' => ['sometimes', 'required', 'string', 'max:255'], - 'screenshot_key' => ['nullable', 'string', 'max:255'], - ]; - } -} diff --git a/app/Concerns/SitemapPageValidationRules.php b/app/Concerns/SitemapPageValidationRules.php deleted file mode 100644 index 82bd0a1..0000000 --- a/app/Concerns/SitemapPageValidationRules.php +++ /dev/null @@ -1,23 +0,0 @@ - */ - protected function sitemapPageRules(): array - { - return [ - 'domain' => ['nullable', 'string', 'max:255'], - 'meta' => ['nullable', 'array'], - 'pageable_id' => ['sometimes', 'required', 'integer', 'exists:sitemaps,id'], - 'pageable_type' => ['sometimes', 'required', 'string', Rule::in(Sitemap::class)], - 'screenshot_key' => ['nullable', 'string', 'ends_with:png'], - 'title' => ['required', 'string', 'max:1025'], - 'url' => ['required', 'string', 'url'], - ]; - } -} diff --git a/app/Concerns/SitemapValidationRules.php b/app/Concerns/SitemapValidationRules.php deleted file mode 100644 index 8699c74..0000000 --- a/app/Concerns/SitemapValidationRules.php +++ /dev/null @@ -1,17 +0,0 @@ - */ - protected function sitemapRules(): array - { - return [ - 'bot_id' => ['sometimes', 'required', 'integer', 'exists:bots,id'], - 'domain' => ['sometimes', 'required', 'string', 'max:255'], - 'urls' => ['nullable', 'array'], - 'urls.*' => ['required', 'string', 'url'], - ]; - } -} diff --git a/app/Console/Commands/GoPub.php b/app/Console/Commands/GoPub.php new file mode 100644 index 0000000..620be8e --- /dev/null +++ b/app/Console/Commands/GoPub.php @@ -0,0 +1,20 @@ +info("Sending bot ({$this->argument('id')}) to the go engine."); + Go::pub($this->argument('id')); + $this->info('Sent.'); + } +} diff --git a/app/Console/Commands/GoSub.php b/app/Console/Commands/GoSub.php new file mode 100644 index 0000000..e4fa271 --- /dev/null +++ b/app/Console/Commands/GoSub.php @@ -0,0 +1,20 @@ +info('Listening to the go engine for events...'); + Go::sub(fn (string $message) => $this->info("Received:\n$message")); + $this->info('Done listening.'); + } +} diff --git a/app/Enums/NewsSource.php b/app/Enums/NewsSource.php deleted file mode 100644 index 8a191dd..0000000 --- a/app/Enums/NewsSource.php +++ /dev/null @@ -1,9 +0,0 @@ - $message) + */ +class Go extends Facade +{ + protected static function getFacadeAccessor(): string + { + return GoService::class; + } +} diff --git a/app/Facades/Rss.php b/app/Facades/Rss.php deleted file mode 100644 index a9c2123..0000000 --- a/app/Facades/Rss.php +++ /dev/null @@ -1,18 +0,0 @@ - $fields = []) - * @method static Closure enqueueFN(array $fields = []) - */ -class Sqs extends Facade -{ - protected static function getFacadeAccessor(): string - { - return SqsService::class; - } -} diff --git a/app/Http/Controllers/Api/ScrapeJobController.php b/app/Http/Controllers/Api/ScrapeJobController.php deleted file mode 100644 index 46ea582..0000000 --- a/app/Http/Controllers/Api/ScrapeJobController.php +++ /dev/null @@ -1,110 +0,0 @@ -input('url'), - $request->input('content_key'), - ); - - $serp->update([ - 'data' => $parsed->data(), - 'screenshot_key' => $request->input('screenshot_key'), - 'content_key' => $parsed->contentKey, - ]); - - if ($parsed->blocked()) { - Log::warning('ScrapeJob: worker reported a Google block/CAPTCHA page', [ - 'serp_id' => $serp->id, - 'query' => $serp->query, - ]); - } - - return response()->noContent(); - } - - public function news(PageSaveRequest $request, Article $article): Response - { - $page = new NewsPage( - $request->input('url'), - $request->input('content_key'), - ); - - (new UpdateArticle)( - $article, - $page->body(), - $page->description(), - $page->imgAlt(), - $page->imgSrc(), - $page->keywords(), - ); - - return response()->noContent(); - } - - public function sitemap(PageSaveRequest $request, Bot $bot): Response - { - $page = new SitemapPage( - $request->input('url'), - $request->input('content_key'), - ); - - (new UpdateOrCreateSitemapPage)( - $bot, - $page->url, - $page->title(), - $request->input('screenshot_key'), - $page->meta(), - ); - - if (! isset($bot->sitemap->urls[$page->url])) { - $urls = $bot->sitemap->urls; - $urls[] = [$page->url => true]; - $bot->sitemap->urls = $urls; - (new UpdateSitemapUrls)($bot->sitemap); - } - - $links = $page->links(); - if ($links->isEmpty()) { - return response()->noContent(); - } - - if ($request->integer('depth') < 0) { - $urls = $bot->sitemap->urls; - foreach ($links->all() as $link) { - $urls[] = [$link => false]; - } - $bot->sitemap->urls = $urls; - (new UpdateSitemapUrls)($bot->sitemap); - return response()->noContent(); - } - - $links->transform(fn (string $link) => [ - 'type' => $bot->type, - 'id' => $bot->id, - 'url' => $link, - 'depth' => $request->integer('depth') - 1, - ])->each(Sqs::enqueueFN()); - - return response()->noContent(); - } -} diff --git a/app/Http/Requests/Api/PageSaveRequest.php b/app/Http/Requests/Api/PageSaveRequest.php deleted file mode 100644 index d9aa577..0000000 --- a/app/Http/Requests/Api/PageSaveRequest.php +++ /dev/null @@ -1,17 +0,0 @@ - */ - public function rules(): array - { - return $this->scrapeRules(); - } -} diff --git a/app/Jobs/BotJob.php b/app/Jobs/BotJob.php index 871f79b..fe87542 100644 --- a/app/Jobs/BotJob.php +++ b/app/Jobs/BotJob.php @@ -2,23 +2,13 @@ namespace App\Jobs; -use App\Actions\Model\UpdateBotRunTimestamp; -use App\Enums\BotType; -use App\Events\BotResultsPersisted; -use App\Facades\Rss; -use App\Facades\Sqs; -use App\Models\Article; +use App\Facades\Go; use App\Models\Bot; -use App\Support\Rss\RssItem; use Illuminate\Bus\Batchable; -use App\Models\Proxy; -use App\Support\Rss\BaseRssItem; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Queue\Queueable; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Context; use Illuminate\Support\Facades\Log; use Throwable; @@ -32,7 +22,7 @@ public function __construct( public function uniqueId(): string { - return Context::id(); + return "bot:{$this->bot->id}"; } public function prepareForDispatch(): bool @@ -43,77 +33,11 @@ public function prepareForDispatch(): bool public function handle(): void { Log::debug('BotJob'); - - switch ($this->bot->type) { - case BotType::Search: - Sqs::enqueue(Context::all()); - break; - case BotType::Sitemap: - Sqs::enqueue([...Context::all(), ...['url' => "https://{$this->bot->query}", 'depth' => 5]]); - break; - case BotType::News: - collect(Rss::news($this->bot->query)) - ->filter(fn (RssItem $item) => $this->bot->lastRunAt()->isBefore($item->publishedAt())) - ->reject(fn (RssItem $item) => $this->bot->blacklisted($item->title(), $item->description())) - ->transform(fn (RssItem $item) => $this->bot->articles()->updateOrCreate( - attributes: ['url' => $item->url()], - values: $item->toArray(), - ))->each(fn (Article $article) => Sqs::enqueue([ - 'type' => BotType::News->value, - 'id' => $this->bot->id, - 'url' => $article->url, - ]))->whenNotEmpty(function (Collection $collection) { - BotResultsPersisted::dispatch($this->bot, __(':count new article(s) queued for ":query".', [ - 'count' => $collection->count(), - 'query' => $this->bot->query, - ])); - }); - } - (new UpdateBotRunTimestamp)($this->bot); - } - - public function handleSearch(): void - { - Sqs::enqueueScrape('serp', $this->bot->serp->id, ['query' => $this->bot->query]); - } - - public function handleSitemap(): void - { - $root = "https://{$this->bot->query}"; - - Sqs::enqueueScrape('sitemap', $this->bot->id, ['url' => $root, 'depth' => 5]); - } - - /** - * @return array> - */ - protected function proxyFields(): array - { - $proxy = $this->bot->randomProxy(); - - if ($proxy === null) { - return []; - } - - return ['proxy' => $this->proxyPayload($proxy)]; - } - - /** @return array */ - protected function proxyPayload(Proxy $proxy): array - { - return [ - 'scheme' => $proxy->scheme, - 'host' => $proxy->host, - 'port' => $proxy->port, - 'username' => $proxy->username, - 'pass' => $proxy->pass, - 'bypass' => $proxy->bypass, - ]; + Go::pub($this->bot); } public function failed(?Throwable $e): void { Log::error('BotJob', ['exception' => $e]); - (new UpdateBotRunTimestamp)($this->bot); } } diff --git a/app/Models/Article.php b/app/Models/Article.php index 4f42d7f..684dcb2 100644 --- a/app/Models/Article.php +++ b/app/Models/Article.php @@ -108,7 +108,7 @@ public static function row(): Closure } if (blank($source)) { - $source = URL::toDomain($article->url); + $source = URL::domain($article->url); } return [ diff --git a/app/Models/Bot.php b/app/Models/Bot.php index ebe823d..81058f2 100644 --- a/app/Models/Bot.php +++ b/app/Models/Bot.php @@ -183,4 +183,13 @@ public function blacklisted(string ...$args): bool $map = array_map(fn (string $item) => [trim($item) => true], $arr); return array_any($this->blacklist(), fn (string $key) => isset($map[$key])); } + + public function childId(): ?int + { + return match ($this->type) { + BotType::Search => $this->serp->id ?? 0, + BotType::Sitemap => $this->sitemap->id ?? 0, + default => -1, + }; + } } diff --git a/app/Observers/SerpObserver.php b/app/Observers/SerpObserver.php index 854d59c..51c6446 100644 --- a/app/Observers/SerpObserver.php +++ b/app/Observers/SerpObserver.php @@ -2,15 +2,15 @@ namespace App\Observers; +use App\Facades\S3; use App\Models\Serp; -use Illuminate\Support\Facades\Storage; class SerpObserver { public function deleting(Serp $model): void { if ($model->content_key) { - Storage::disk('s3')->delete($model->content_key); + S3::del($model->content_key); } $model->deleteScreenshot(); $model->deletePages(); diff --git a/app/Providers/MacroServiceProvider.php b/app/Providers/MacroServiceProvider.php index de65f79..e86eb9b 100644 --- a/app/Providers/MacroServiceProvider.php +++ b/app/Providers/MacroServiceProvider.php @@ -4,24 +4,18 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Arr; -use Illuminate\Support\Facades\Context; -use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\URL; +use Illuminate\Support\Fluent; use Illuminate\Support\ServiceProvider; use Illuminate\Support\Str; -use Illuminate\Support\Stringable; -use Illuminate\Support\Uri; -use Ramsey\Uuid\Uuid; class MacroServiceProvider extends ServiceProvider { public function boot(): void { $this->configureArr(); - $this->configureHttp(); - $this->configureUri(); $this->configureURL(); - $this->configureContext(); + $this->configureFluent(); } private function configureArr(): void @@ -48,17 +42,7 @@ private function configureArr(): void private function configureURL(): void { - URL::macro('toDomain', function (?string $url) { - if (str($url)->substrCount('.') === 1) { - return $url; - } - return $url !== null - ? parse_url($url, PHP_URL_HOST) - |> (fn ($x) => explode('.', (string) $x)) - |> (fn ($x) => array_slice($x, -2)) - |> (fn ($x) => implode('.', $x)) - : ''; - }); + URL::macro('clean', fn (?string $url): string => str($url)->trim()->rtrim('/')->toString()); URL::macro('domain', function (?string $url): string { return $url === null ? '' : parse_url($url, PHP_URL_HOST) @@ -71,37 +55,14 @@ private function configureURL(): void if ($url === null) { $url = 'bytelyon.com'; } else { - $url = URL::toDomain($url); + $url = URL::domain($url); } return "https://www.google.com/s2/favicons?domain=$url&sz=$size"; }); - - URL::macro('toUuid5', function (?string $url): ?string { - if ($url === null) { - return null; - } - return Uuid::uuid5(Uuid::NAMESPACE_URL, $url)->toString(); - }); - } - - private function configureHttp(): void - { - Http::macro('withProxy', fn (Stringable $proxy) => $this->withOptions(['proxy' => (string) $proxy])); - } - - private function configureUri(): void - { - Uri::macro('clean', fn (?string $url): string => str($url)->trim()->rtrim('/')->toString()); - Uri::macro('domain', function (?string $url): string { - return $url === null ? '' : parse_url($url, PHP_URL_HOST) - |> (fn ($x) => explode('.', (string) $x)) - |> (fn ($x) => array_slice($x, -2)) - |> (fn ($x) => implode('.', $x)); - }); } - private function configureContext(): void + private function configureFluent(): void { - Context::macro('id', fn (): string => rescue(fn () => collect(Context::all())->join(':'), Str::ulid())); + Fluent::macro('of', fn(string $value): Fluent => new Fluent(json_decode($value, true))); } } diff --git a/app/Services/GoService.php b/app/Services/GoService.php new file mode 100644 index 0000000..d88cea3 --- /dev/null +++ b/app/Services/GoService.php @@ -0,0 +1,61 @@ +client = Redis::connection('go'); + } + + public function pub(Bot|int $bot): void { + if (is_int($bot)) { + try { + $bot = Bot::query()->findOrFail($bot); + } catch (ModelNotFoundException $e) { + return; + } + } + + $this->client->publish('bots', json_encode([ + 'id' => $bot->id, + 'child_id' => $bot->childId(), + 'type' => $bot->type->value, + 'headless' => $bot->headless, + 'query' => $bot->query, + 'last_run_at' => $bot->lastRunAt(), + 'blacklist' => $bot->blacklist(), + ])); + } + + public function sub(Closure $callback): void { + $this->client->subscribe('evts', function (string $message) use ($callback) { + $callback($message); + $ƒ = Fluent::of($message); + try { + BotResultsPersisted::dispatch( + Bot::query()->findOrFail($ƒ->integer('id')), + $ƒ->string('message'), + ); + } catch (ModelNotFoundException $e) { + Log::error('GoService#subscribe', [ + 'channel' => 'evts', + 'message' => $message, + 'exception' => $e, + ]); + } + }); + } +} diff --git a/app/Services/RssService.php b/app/Services/RssService.php deleted file mode 100644 index a50145a..0000000 --- a/app/Services/RssService.php +++ /dev/null @@ -1,54 +0,0 @@ -bing($query), - $this->google($query), - ); - } - - /** @return BingRssItem[] */ - protected function bing(string $query): array - { - return $this->items(BingRssItem::class, 'https://www.bing.com/news/search', [ - 'q' => $query, - 'format' => 'rss', - ]); - } - - /** @return GoogleRssItem[] */ - protected function google(string $query): array - { - return $this->items(GoogleRssItem::class, 'https://news.google.com/rss/search', [ - 'q' => $query, - 'hl' => 'en-US', - 'gl' => 'US', - 'ceid' => 'US:en', - ]); - } - - /** @return RssItem[] */ - private function items(string $class, string $url, array $query): array - { - return rescue(function () use ($class, $url, $query) { - $xml = simplexml_load_string( - data: Http::get($url, $query)->body(), - class_name: $class, - )->xpath('//item'); - return ! is_bool($xml) ? (array) $xml : []; - }, [], false); - } -} diff --git a/app/Services/S3Service.php b/app/Services/S3Service.php new file mode 100644 index 0000000..b58af11 --- /dev/null +++ b/app/Services/S3Service.php @@ -0,0 +1,44 @@ +client = Storage::disk('s3'); + } + + public function del(?string $key): bool { + if (App::runningUnitTests()) { + return false; + } + if ($key === null) { + Log::warning("S3Service::del() called with null key"); + return false; + } + return $this->client->delete($key); + } + + public function url(?string $key, Closure|DateTimeInterface|DateInterval|int $ttl = 900): ?string { + if ($key === null) { + Log::warning("S3Service::url() called with null key"); + return null; + } + return cache()->remember( + "s3:url:$key", + $ttl, + fn() => $this->client->temporaryUrl($key, now()->addSeconds($ttl)) + ); + } +} diff --git a/app/Services/SqsService.php b/app/Services/SqsService.php deleted file mode 100644 index 6fe3473..0000000 --- a/app/Services/SqsService.php +++ /dev/null @@ -1,43 +0,0 @@ -client = new SqsClient(config('services.sqs')); - $this->queueUrl = config('services.sqs.scrape_jobs_queue_url'); - } - - /** @param array $payload */ - public function enqueue(array $payload = []): void - { - $this->enqueueFN()($payload); - } - - public function enqueueFN(): Closure - { - if (App::runningUnitTests()) { - return fn () => []; - } - - /** @param array $payload */ - return fn (array $payload = []) => $this->client->sendMessage([ - 'QueueUrl' => $this->queueUrl, - 'MessageBody' => json_encode($payload), - ]); - } -} diff --git a/app/Support/Rss/BingRssItem.php b/app/Support/Rss/BingRssItem.php deleted file mode 100644 index 7e96033..0000000 --- a/app/Support/Rss/BingRssItem.php +++ /dev/null @@ -1,29 +0,0 @@ -xpath('//News:Source'), default: ''); - } - - public function url(): string - { - foreach (explode('&', parse_url((string) $this->link)['query']) as $param) { - $keyValue = explode('=', $param); - if ($keyValue[0] === 'url') { - return rtrim(urldecode($keyValue[1]), '/'); - } - } - return (string) $this->link; - } - - public function imageSrc(): string - { - return (string) Arr::first(array: $this->xpath('//News:Image'), default: ''); - } -} diff --git a/app/Support/Rss/GoogleRssItem.php b/app/Support/Rss/GoogleRssItem.php deleted file mode 100644 index 7918094..0000000 --- a/app/Support/Rss/GoogleRssItem.php +++ /dev/null @@ -1,183 +0,0 @@ -[^?]+)~'; - - private const string USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' - .'(KHTML, like Gecko) Chrome/124.0 Safari/537.36'; - - private const int RETRIES = 2; - - private const int TIMEOUT = 30; - - public function imageSrc(): string - { - return ''; - } - - public function publisher(): string - { - return (string) $this->source; - } - - public function title(): string - { - return rescue(fn () => explode(' - ', $this->title)[0], (string) $this->title); - } - - public function url(): string - { - return $this->decoded((string) $this->link); - } - - private function decoded(string $link): string - { - $key = 'gnews:decoded:'.sha1($link); - if (is_string($cached = Cache::get($key))) { - return $cached; - } - - $url = $this->decode($link); - if (! empty($url)) { - Cache::put($key, $url, now()->addDay()); - } - return $url; - } - - private function decode(string $link): ?string - { - $context = ['gURL' => $link]; - - if (preg_match(self::LINK_REGEX, $link, $matches) !== 1) { - Log::warning('failed to match gstatic regex', $context); - - return ''; - } - - try { - $node = HTMLDocument::createFromString( - source: $this->client()->get($link)->body(), - options: self::LIBXML_NOERROR, - ); - } catch (Throwable $e) { - Log::warning('failed to parse gstatic html', [...$context, 'error' => $e->getMessage()]); - - return ''; - } - - $encoded = $matches['encoded_url']; - $url = ''; - - try { - $url = $this->decodeNode($node, $encoded); - // @codeCoverageIgnoreStart - } catch (Throwable $e) { - Log::warning('failed to decode gstatic node', [...$context, 'error' => $e->getMessage()]); - - return ''; - } - // @codeCoverageIgnoreEnd - - Log::debug('decoded gstatic url', ['url' => $url]); - return rtrim($url, '/'); - } - - private function decodeNode(HTMLDocument $node, string $encodedText): ?string - { - /** - * @return array{0: string, 1: string} [signature, timestamp] - */ - $ƒ = function (Element $wiz): array { - return [ - $wiz->firstElementChild?->getAttribute('data-n-a-sg') ?? '', - $wiz->firstElementChild?->getAttribute('data-n-a-ts') ?? '', - ]; - }; - - foreach ($node->getElementsByTagName('c-wiz') as $wiz) { - [$sg, $ts] = $ƒ($wiz); - - try { - return $this->decodeParts($sg, $ts, $encodedText); - } catch (Throwable) { - } - } - return null; - } - - /** - * @throws JsonException - * @throws ConnectionException - */ - protected function decodeParts(string $signature, string $timestamp, string $base64Str): ?string - { - if ($signature === '' || $timestamp === '') { - throw new RuntimeException('missing signature or timestamp'); - } - - $request = sprintf( - '["garturlreq",[["X","X",["X","X"],null,null,1,1,"US:en",null,1,null,null,null,null,null,0,1],' - .'"X","X",1,[1,1,1],1,1,null,0,0,null,0],"%s",%s,"%s"]', - $base64Str, - $timestamp, - $signature, - ); - - $body = json_encode([[['Fbv4je', $request]]], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); - - $raw = $this->client() - ->asForm() - ->post(self::ENDPOINT, ['f.req' => $body]) - ->body(); - - $line = Str::of($raw)->explode("\n\n")->get(1); - if (! is_string($line)) { - throw new RuntimeException('unexpected batch-execute response format'); - } - - $payload = json_decode($line, true, 512, JSON_THROW_ON_ERROR); - if (! is_array($payload) || $payload === []) { - throw new RuntimeException('empty payload'); - } - - $innerJson = data_get($payload, '0.2'); - if (! is_string($innerJson)) { - throw new RuntimeException('missing inner json string'); - } - - $url = data_get(json_decode($innerJson, true, 512, JSON_THROW_ON_ERROR), 1); - if (! is_string($url) || $url === '') { - throw new RuntimeException('decoded url not string'); - } - - return $url; - } - - private function client(): PendingRequest - { - return Http::withUserAgent(self::USER_AGENT) - ->connectTimeout(5) - ->timeout(self::TIMEOUT) - ->retry(self::RETRIES, 250, throw: false) - ->throw(); - } -} diff --git a/app/Support/Rss/RssItem.php b/app/Support/Rss/RssItem.php deleted file mode 100644 index bf9825a..0000000 --- a/app/Support/Rss/RssItem.php +++ /dev/null @@ -1,57 +0,0 @@ -description; - } - - public function title(): string - { - return (string) $this->title; - } - - public function url(): string - { - return (string) $this->link; - } - - public function publishedAt(): CarbonInterface|string - { - return rescue( - fn () => Carbon::parse((string) $this->pubDate), - fn () => (string) $this->pubDate, - fn (InvalidArgumentException $e) => Log::warning("RssItem#publishedAt: {$e->getMessage()}")); - } - - /** @return array */ - public function toArray(): array - { - $publishedAt = $this->publishedAt(); - return [ - 'description' => $this->description(), - 'img_src' => $this->imageSrc(), - 'published_at' => is_string($publishedAt) ? $publishedAt : $publishedAt->toDateTimeString(), - 'publisher' => $this->publisher(), - 'source' => $this->source(), - 'title' => $this->title(), - 'url' => $this->url(), - ]; - } -} diff --git a/app/Support/Sqs/Html.php b/app/Support/Sqs/Html.php deleted file mode 100644 index ae0ed10..0000000 --- a/app/Support/Sqs/Html.php +++ /dev/null @@ -1,63 +0,0 @@ -document = rescue( - fn () => HTMLDocument::createFromString( - source: $this->content ?? '', - options: LIBXML_NOERROR | LIBXML_HTML_NOIMPLIED, - ), HTMLDocument::createEmpty(), - ); - $this->emptyElement = $this->document->createElement('div'); - } - - public function title(): string - { - return $this->document->title; - } - - public function attr(string $selector, string $name): Stringable - { - return Str::of($this->element($selector)->getAttribute($name)); - } - - public function element(string $selector): Element - { - return $this->document->querySelector($selector) ?? $this->emptyElement; - } - - /** - * @implements IteratorAggregate - * - * @return NodeList - */ - public function elements(string $selector): NodeList - { - return $this->document->querySelectorAll($selector); - } - - public function isMissing(string $selector): bool - { - return $this->document->querySelector($selector) === null; - } -} diff --git a/app/Support/Sqs/Page.php b/app/Support/Sqs/Page.php deleted file mode 100644 index 20fb658..0000000 --- a/app/Support/Sqs/Page.php +++ /dev/null @@ -1,42 +0,0 @@ -url = Uri::clean($this->rawUrl); - $this->domain = Uri::domain($this->url); - } - - /** - * @return array - */ - public function toArray(): array - { - return [ - 'body' => $this->body(), - 'description' => $this->description(), - 'domain' => $this->domain, - 'img_alt' => $this->imgAlt(), - 'img_url' => $this->imgSrc(), - 'keywords' => $this->keywords(), - 'title' => $this->title(), - 'url' => $this->url, - ]; - } -} diff --git a/app/Support/Sqs/Serp.php b/app/Support/Sqs/Serp.php deleted file mode 100644 index dfa216f..0000000 --- a/app/Support/Sqs/Serp.php +++ /dev/null @@ -1,191 +0,0 @@ -url, '/sorry/') - || str($this->content)->contains(self::BLOCK_PHRASES, true); - } - - /** @return array>> */ - public function data(): array - { - if ($this->blocked()) { - return $this->emptyData(); - } - - return [ - 'sponsored_results' => $this->sponsoredResults(), - 'sponsored_products' => $this->sponsoredProducts(), - 'organic_results' => $this->organicResults(), - 'organic_products' => $this->organicProducts(), - 'similar_queries' => $this->similarQueries(), - ]; - } - - /** @return array>> */ - public function emptyData(): array - { - return [ - 'sponsored_results' => [], - 'sponsored_products' => [], - 'organic_results' => [], - 'organic_products' => [], - 'similar_queries' => [], - ]; - } - - /** @return array> */ - private function sponsoredResults(): array - { - return collect($this->elements('[data-pcu]')) - ->values() - ->map(function (Element $e, int $i) { - $spans = $e->querySelectorAll('span'); - $title = $spans->item(0)?->textContent ?? ''; - $rawBrand = trim($spans->item(1)?->textContent ?? ''); - $brand = trim(explode('https://', $rawBrand)[0]); - $url = $this->resolveUrl($e->getAttribute('href') ?? ''); - - return [ - 'kind' => 'sponsored_result', - 'index' => $i, - 'title' => trim($title), - 'brand' => $brand, - 'url' => $url, - 'domain' => Uri::domain($url), - ]; - }) - ->all(); - } - - /** @return array> */ - private function organicProducts(): array - { - return collect($this->elements('product-viewer-entrypoint')) - ->values() - ->map(function (Element $e, int $i) { - $div = $e->querySelector('div'); - $img = $e->querySelector('img'); - $title = $div?->getAttribute('aria-label') ?? ''; - if (blank($title) && $img) { - $title = $img->getAttribute('alt') ?? ''; - } - - return [ - 'kind' => 'organic_product', - 'index' => $i, - 'title' => trim($title), - 'image' => $img?->getAttribute('src') ?? '', - ]; - }) - ->all(); - } - - /** @return array> */ - private function sponsoredProducts(): array - { - return collect($this->elements('[data-dtld]')) - ->values() - ->map(function (Element $e, int $i) { - $div = $e->querySelector('div.pla-unit-container'); - $a = $div?->querySelector('a.pla-unit-img-container-link'); - $titleEl = $div?->querySelector("div[data-call_grow_wiz_event='true']"); - $priceEl = $div?->querySelector('div[aria-label]'); - $brandEl = $div?->querySelector("span[role='text']"); - $img = $a?->querySelector('img'); - $url = $this->resolveUrl($a?->getAttribute('href') ?? ''); - - return [ - 'kind' => 'sponsored_product', - 'index' => $i, - 'domain' => $e->getAttribute('data-dtld') ?? '', - 'url' => $url, - 'image' => $img?->getAttribute('src') ?? '', - 'title' => trim($titleEl?->textContent ?? ''), - 'price' => $priceEl?->getAttribute('aria-label') ?? '', - 'brand' => trim($brandEl?->textContent ?? ''), - ]; - }) - ->all(); - } - - /** @return array> */ - private function organicResults(): array - { - return collect($this->elements('h3[id]')) - ->values() - ->map(function (Element $e, int $i) { - $parent = $e->parentElement; - $url = $this->resolveUrl($parent?->getAttribute('href') ?? ''); - - return [ - 'kind' => 'organic_result', - 'index' => $i, - 'title' => trim($e->textContent ?? ''), - 'url' => $url, - 'domain' => Uri::domain($url), - ]; - }) - ->all(); - } - - /** @return array> */ - private function similarQueries(): array - { - /** @var Collection $values */ - $values = collect($this->elements('div[data-notify-expansion]')) - ->map(fn (Element $e) => trim($e->getAttribute('data-q') ?? '')) - ->filter(fn (string $q) => strlen($q) > 4); - - $botstuff = $this->element('div#botstuff'); - if ($botstuff !== null) { - $values = $values->merge( - collect($botstuff->querySelectorAll('a')) - ->map(fn (Element $e) => trim($e->textContent ?? '')) - ->filter(fn (string $t) => strlen($t) > 4) - ); - } - - return $values->values() - ->map(fn (string $value, int $i) => ['kind' => 'similar_query', 'index' => $i, 'value' => $value]) - ->all(); - } - - private function resolveUrl(string $href): string - { - if ($href === '') { - return $href; - } - - if (! str_starts_with($href, 'https://www.google.com')) { - if (! str_starts_with($href, '/')) { - $href = "/$href"; - } - $href = "https://www.google.com$href"; - } - - $query = []; - parse_str((string) parse_url($href, PHP_URL_QUERY), $query); - if (isset($query['q']) && str_starts_with($query['q'], 'http')) { - return $query['q']; - } - if (isset($query['url']) && str_starts_with($query['url'], 'http')) { - return $query['url']; - } - - return $href; - } -} diff --git a/app/Traits/HasBody.php b/app/Traits/HasBody.php deleted file mode 100644 index 4d81a08..0000000 --- a/app/Traits/HasBody.php +++ /dev/null @@ -1,27 +0,0 @@ -isMissing($tag)) { - continue; - } - - $values = collect($this->element($tag)->querySelectorAll('p')) - ->transform(fn (Element $e): string => trim($e->textContent ?? '')) - ->reject(fn (string $href): bool => empty($href)); - - if ($values->isNotEmpty()) { - return $values->join(' '); - } - } - - return ''; - } -} diff --git a/app/Traits/HasLinks.php b/app/Traits/HasLinks.php deleted file mode 100644 index f526881..0000000 --- a/app/Traits/HasLinks.php +++ /dev/null @@ -1,66 +0,0 @@ - - */ - public function links(): Collection - { - return $this - ->anchorElements() - ->transform(self::toHref()) - ->reject(self::deadEnds()) - ->transform(self::toUri($this->url)) - ->filter(self::sameSite($this->domain)) - ->transform(self::toString()) - ->unique() - ->values(); - } - - private function anchorElements(): Collection - { - return collect($this->elements('a')); - } - - private static function toHref(): callable - { - return function (Element $element): Stringable { - return str($element->getAttribute('href'))->trim(); - }; - } - - private static function deadEnds(): callable - { - return function (Stringable $href): bool { - return $href->isEmpty() - || $href->length() === 1 - || $href->endsWith('#') - || $href->startsWith('http://'); - }; - } - - private static function toUri(string $url): callable - { - return fn (Stringable $href): Uri => $href - ->whenStartsWith('/', fn (): Stringable => $href->prepend($url)) - ->toUri(); - } - - private static function sameSite(string $domain): callable - { - return fn (Uri $uri): bool => str_ends_with($uri->host(), $domain); - } - - private static function toString(): callable - { - return fn (Uri $uri): string => $uri->value(); - } -} diff --git a/app/Traits/HasMeta.php b/app/Traits/HasMeta.php deleted file mode 100644 index 6500b5e..0000000 --- a/app/Traits/HasMeta.php +++ /dev/null @@ -1,93 +0,0 @@ - $metaKeys - */ - private function metaVal(array $metaKeys, bool $allVals = false): string - { - /** @var array $arr */ - $arr = []; - foreach ($metaKeys as $key) { - $attr = $this->attr("meta[$key]", 'content')->trim(); - if ($attr->isEmpty()) { - continue; - } - if (! $allVals) { - return $attr->value(); - } - $arr = [...$arr, ...$attr->explode(',')]; - } - return collect($arr) - ->transform(fn (string $val): string => trim($val)) - ->filter() - ->join(','); - } - - /** @return array */ - public function meta(): array - { - $meta = []; - foreach ($this->elements('meta') as $e) { - - $val = str($e->getAttribute('content'))->trim(); - if ($val->isEmpty()) { - continue; - } - - $key = value(function (?string $name, ?string $property): string { - return str($name)->trim()->whenEmpty(fn ($str) => $str->append($property))->trim(); - }, $e->getAttribute('name'), $e->getAttribute('property')); - - if (! empty($key) && ! isset($meta[$key])) { - $meta[$key] = $val->value(); - } - } - return $meta; - } - - public function description(): string - { - return $this->metaVal(self::descriptionMetaKeys); - } - - public function imgAlt(): string - { - return $this->metaVal(self::imgAltMetaKeys); - } - - public function imgSrc(): string - { - return $this->metaVal(self::imgSrcMetaKeys); - } - - /** - * @return array - */ - public function keywords(): array - { - return explode(',', $this->metaVal(self::keywordMetaKeys, true)); - } -} diff --git a/app/Traits/HasNewsSource.php b/app/Traits/HasNewsSource.php deleted file mode 100644 index b2ce8f7..0000000 --- a/app/Traits/HasNewsSource.php +++ /dev/null @@ -1,20 +0,0 @@ - NewsSource::BingNews->value, - GoogleRssItem::class => NewsSource::GoogleNews->value, - default => RssItem::class, - }; - } -} diff --git a/app/Traits/HasScreenshot.php b/app/Traits/HasScreenshot.php index 7315eb4..266007e 100644 --- a/app/Traits/HasScreenshot.php +++ b/app/Traits/HasScreenshot.php @@ -2,30 +2,17 @@ namespace App\Traits; -use Illuminate\Support\Facades\App; -use Illuminate\Support\Facades\Storage; +use App\Facades\S3; trait HasScreenshot { public function screenshotUrl(): ?string { - if ($this->screenshot_key === null) { - return null; - } - - $ttl = now()->addMinutes(15); - - return cache()->remember( - $this->screenshot_key, - $ttl, - fn () => Storage::disk('s3')->temporaryUrl($this->screenshot_key, $ttl), - ); + return S3::url($this->screenshot_key); } public function deleteScreenshot(): void { - if ($this->screenshot_key !== null && ! App::runningUnitTests()) { - Storage::disk('s3')->delete($this->screenshot_key); - } + S3::del($this->screenshot_key); } } diff --git a/app/Traits/Services/HasSqsClient.php b/app/Traits/Services/HasSqsClient.php deleted file mode 100644 index ba8edad..0000000 --- a/app/Traits/Services/HasSqsClient.php +++ /dev/null @@ -1,5 +0,0 @@ - env('REDIS_BACKOFF_CAP', 1000), 'prefix' => '', ], + + 'go' => [ + 'url' => env('REDIS_URL'), + 'host' => env('REDIS_HOST', '127.0.0.1'), + 'username' => env('REDIS_USERNAME'), + 'password' => env('REDIS_PASSWORD'), + 'port' => env('REDIS_PORT', '6379'), + 'database' => env('REDIS_CACHE_DB', '10'), + 'max_retries' => env('REDIS_MAX_RETRIES', 3), + 'backoff_algorithm' => env('REDIS_BACKOFF_ALGORITHM', 'decorrelated_jitter'), + 'backoff_base' => env('REDIS_BACKOFF_BASE', 100), + 'backoff_cap' => env('REDIS_BACKOFF_CAP', 1000), + 'prefix' => '', + ], ], ]; diff --git a/routes/api.php b/routes/api.php index db69df8..5cc3457 100644 --- a/routes/api.php +++ b/routes/api.php @@ -1,12 +1,5 @@ name('api.')->group(function () { - Route::prefix('scrape-jobs')->name('scrape-jobs.')->group(function () { - Route::post('search/{serp}/complete', [ScrapeJobController::class, 'serp'])->name('search.complete'); - Route::post('news/{article}/complete', [ScrapeJobController::class, 'news'])->name('news.complete'); - Route::post('sitemap/{bot}/complete', [ScrapeJobController::class, 'sitemap'])->name('sitemap.complete'); - }); -}); +Route::middleware(['auth:sanctum', 'abilities:worker'])->name('api.')->group(function () {}); diff --git a/tests/Unit/Support/Rss/BaseRssItemTest.php b/tests/Unit/Support/Rss/BaseRssItemTest.php deleted file mode 100644 index adc39ea..0000000 --- a/tests/Unit/Support/Rss/BaseRssItemTest.php +++ /dev/null @@ -1,87 +0,0 @@ - - -Item Title -https://example.com/article -Item description -$pubDate -"; - - return new ConcreteRssItem($xml); - } - - public function test_published_at_parses_valid_date(): void - { - $item = $this->makeItem('Mon, 01 Jan 2024 10:00:00 GMT'); - - $publishedAt = $item->publishedAt(); - - $this->assertInstanceOf(Carbon::class, $publishedAt); - $this->assertSame('2024-01-01 10:00:00', $publishedAt->toDateTimeString()); - } - - public function test_published_at_logs_and_returns_raw_string_on_invalid_date(): void - { - Log::shouldReceive('warning') - ->once() - ->withArgs(fn (string $message): bool => str_starts_with($message, 'RssItem#publishedAt: ')); - - $item = $this->makeItem('not a real date at all'); - - $this->assertSame('not a real date at all', $item->publishedAt()); - } - - public function test_description_title_and_url(): void - { - $item = $this->makeItem('Mon, 01 Jan 2024 10:00:00 GMT'); - - $this->assertSame('Item description', $item->description()); - $this->assertSame('Item Title', $item->title()); - $this->assertSame('https://example.com/article', $item->url()); - } - - public function test_to_array_contains_all_expected_keys(): void - { - $item = $this->makeItem('Mon, 01 Jan 2024 10:00:00 GMT'); - - $this->assertSame([ - 'description' => 'Item description', - 'img_src' => 'https://example.com/image.png', - 'published_at' => '2024-01-01 10:00:00', - 'publisher' => 'Example Publisher', - 'source' => 'example-source', - 'title' => 'Item Title', - 'url' => 'https://example.com/article', - ], $item->toArray()); - } -} diff --git a/tests/Unit/Support/Rss/BingRssItemTest.php b/tests/Unit/Support/Rss/BingRssItemTest.php deleted file mode 100644 index b1c0e6c..0000000 --- a/tests/Unit/Support/Rss/BingRssItemTest.php +++ /dev/null @@ -1,71 +0,0 @@ - - - Bing Article Title - $link - Bing description - Mon, 01 Jan 2024 10:00:00 GMT - SomeImage - {$sourceTag} - -XML); - } - - public function test_publisher_and_image_src_return_first_news_source_node(): void - { - $item = $this->makeItem( - link: 'https://www.bing.com/news/apiclick.aspx?ref=X&url=https%3A%2F%2Fexample.com%2Farticle%2F&cc=us', - sourceTag: 'Example Publisher', - ); - - $this->assertSame('Example Publisher', $item->publisher()); - $this->assertSame('SomeImage', $item->imageSrc()); - } - - public function test_publisher_and_image_src_default_to_empty_string_when_missing(): void - { - $item = $this->makeItem( - link: 'https://www.bing.com/news/apiclick.aspx?ref=X&cc=us', - ); - - $this->assertSame('', $item->publisher()); - $this->assertSame('SomeImage', $item->imageSrc()); - } - - public function test_source_returns_bing_news_enum_value(): void - { - $item = $this->makeItem(link: 'https://www.bing.com/news/apiclick.aspx?ref=X&cc=us'); - - $this->assertSame(NewsSource::BingNews->value, $item->source()); - } - - public function test_url_decodes_url_query_parameter(): void - { - $item = $this->makeItem( - link: 'https://www.bing.com/news/apiclick.aspx?ref=X&url=https%3A%2F%2Fexample.com%2Farticle%2F&cc=us', - ); - - $this->assertSame('https://example.com/article', $item->url()); - } - - public function test_url_falls_back_to_raw_link_when_no_url_query_parameter(): void - { - $link = 'https://www.bing.com/news/apiclick.aspx?ref=X&cc=us'; - - $item = $this->makeItem(link: $link); - - $this->assertSame('https://www.bing.com/news/apiclick.aspx?ref=X&cc=us', $item->url()); - } -} diff --git a/tests/Unit/Support/Rss/GoogleRssItemTest.php b/tests/Unit/Support/Rss/GoogleRssItemTest.php deleted file mode 100644 index fed4a0d..0000000 --- a/tests/Unit/Support/Rss/GoogleRssItemTest.php +++ /dev/null @@ -1,251 +0,0 @@ -decodeParts($signature, $timestamp, $base64Str); - } -} - -class GoogleRssItemTest extends TestCase -{ - private const string ENDPOINT = 'https://news.google.com/_/DotsSplashUi/data/batchexecute'; - - protected function tearDown(): void - { - Http::fake(); - - parent::tearDown(); - } - - private function makeXml(string $link, string $title = 'Article Title - Google News', string $source = 'Example Source'): string - { - return " - -$title -$link -Google description -Mon, 01 Jan 2024 10:00:00 GMT -$source -"; - } - - private function batchExecuteBody(?string $decodedUrl): string - { - $innerJson = json_encode([0, $decodedUrl]); - $payload = [['wrb.fr', 'Fbv4je', $innerJson, null, null, null, 'generic']]; - $line = json_encode($payload); - - return ")]}'\n\n{$line}"; - } - - public function test_image_src_is_always_empty(): void - { - $item = new GoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->assertSame('', $item->imageSrc()); - } - - public function test_publisher_returns_source_element(): void - { - $item = new GoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en', source: 'My Source')); - - $this->assertSame('My Source', $item->publisher()); - } - - public function test_title_strips_suffix_after_dash(): void - { - $item = new GoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en', title: 'Real Title - Google News')); - - $this->assertSame('Real Title', $item->title()); - } - - public function test_source_returns_google_news_enum_value(): void - { - $item = new GoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->assertSame(NewsSource::GoogleNews->value, $item->source()); - } - - public function test_url_returns_empty_string_when_link_has_no_articles_segment(): void - { - $link = 'https://news.google.com/rss/topics/SOMETOPIC?hl=en'; - Cache::forget('gnews:decoded:'.sha1($link)); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('', $item->url()); - Http::assertNothingSent(); - } - - public function test_url_returns_empty_string_when_http_request_fails(): void - { - $link = 'https://news.google.com/rss/articles/ENCFAIL?hl=en'; - Cache::forget('gnews:decoded:'.sha1($link)); - - Http::fake([ - 'https://news.google.com/rss/articles/*' => function () { - throw new ConnectionException('connection failed'); - }, - ]); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('', $item->url()); - } - - public function test_url_returns_empty_string_when_page_has_no_c_wiz_element(): void - { - $link = 'https://news.google.com/rss/articles/ENCNOWIZ?hl=en'; - Cache::forget('gnews:decoded:'.sha1($link)); - - Http::fake(callback: [ - 'https://news.google.com/rss/articles/*' => Http::response( - '

no wiz here

', - 200, - ), - ]); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('', $item->url()); - } - - public function test_url_returns_empty_string_when_c_wiz_is_missing_signature_data(): void - { - $link = 'https://news.google.com/rss/articles/ENCEMPTYWIZ?hl=en'; - Cache::forget('gnews:decoded:'.sha1($link)); - - Http::fake([ - 'https://news.google.com/rss/articles/*' => Http::response( - '
', - 200, - ), - ]); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('', $item->url()); - } - - public function test_url_decodes_and_caches_successfully(): void - { - $link = 'https://news.google.com/rss/articles/ENCSUCCESS?hl=en'; - $cacheKey = 'gnews:decoded:'.sha1($link); - Cache::forget($cacheKey); - - Http::fake([ - 'https://news.google.com/rss/articles/*' => Http::response( - '
', - 200, - ), - self::ENDPOINT => Http::response($this->batchExecuteBody('https://example.com/decoded-article/'), 200), - ]); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('https://example.com/decoded-article', $item->url()); - $this->assertSame('https://example.com/decoded-article', Cache::get($cacheKey)); - - Http::assertSentCount(2); - - $this->assertSame('https://example.com/decoded-article', $item->url()); - Http::assertSentCount(2); - - Cache::forget($cacheKey); - } - - public function test_url_returns_cached_value_without_any_http_calls(): void - { - $link = 'https://news.google.com/rss/articles/ENCCACHED?hl=en'; - $cacheKey = 'gnews:decoded:'.sha1($link); - Cache::put($cacheKey, 'https://cached.example.com/article', now()->addDay()); - - Http::fake(); - - $item = new GoogleRssItem($this->makeXml($link)); - - $this->assertSame('https://cached.example.com/article', $item->url()); - Http::assertNothingSent(); - - Cache::forget($cacheKey); - } - - public function test_decode_parts_throws_when_signature_or_timestamp_missing(): void - { - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('missing signature or timestamp'); - - $item->callDecodeParts('', '', 'base64'); - } - - public function test_decode_parts_throws_when_batch_execute_response_has_no_second_line(): void - { - Http::fake([self::ENDPOINT => Http::response('single-line-only', 200)]); - - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('unexpected batch-execute response format'); - - $item->callDecodeParts('sig', 'ts', 'base64'); - } - - public function test_decode_parts_throws_when_payload_is_empty(): void - { - Http::fake([self::ENDPOINT => Http::response("line0\n\n[]", 200)]); - - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('empty payload'); - - $item->callDecodeParts('sig', 'ts', 'base64'); - } - - public function test_decode_parts_throws_when_inner_json_missing(): void - { - $line = json_encode([['wrb.fr', 'Fbv4je', null]]); - Http::fake([self::ENDPOINT => Http::response("line0\n\n{$line}", 200)]); - - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('missing inner json string'); - - $item->callDecodeParts('sig', 'ts', 'base64'); - } - - public function test_decode_parts_throws_when_decoded_url_is_not_a_string(): void - { - Http::fake([self::ENDPOINT => Http::response($this->batchExecuteBody(null), 200)]); - - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage('decoded url not string'); - - $item->callDecodeParts('sig', 'ts', 'base64'); - } - - public function test_decode_parts_returns_decoded_url_on_success(): void - { - Http::fake([self::ENDPOINT => Http::response($this->batchExecuteBody('https://example.com/direct-decode/'), 200)]); - - $item = new DecodePartsExposingGoogleRssItem($this->makeXml('https://news.google.com/rss/articles/ENC1?hl=en')); - - $this->assertSame('https://example.com/direct-decode/', $item->callDecodeParts('sig', 'ts', 'base64')); - } -} diff --git a/tests/Unit/Support/Sqs/PageTest.php b/tests/Unit/Support/Sqs/PageTest.php deleted file mode 100644 index b76956a..0000000 --- a/tests/Unit/Support/Sqs/PageTest.php +++ /dev/null @@ -1,156 +0,0 @@ -$head$body"); - return 'page/test'; - } - - public function test_title_url_and_domain(): void - { - $page = new Page( - 'https://www.example.com/articles/1', - $this->html('My Great Article - Site Name', ''), - ); - - $this->assertSame('My Great Article - Site Name', $page->title()); - $this->assertSame('https://www.example.com/articles/1', $page->url); - $this->assertSame('example.com', $page->domain); - } - - public function test_meta_extracts_name_and_property_and_skips_invalid_tags(): void - { - $page = new Page('https://example.com', $this->html( - <<<'HTML' - - - - - - HTML, - '', - )); - - $this->assertSame([ - 'description' => 'Named meta value', - 'og:title' => 'Property meta value', - ], $page->meta()); - } - - public function test_description_img_alt_and_img_src_return_first_matching_meta(): void - { - $page = new Page('https://example.com', $this->html( - <<<'HTML' - - - - - HTML, - '', - )); - - $this->assertSame('A description here', $page->description()); - $this->assertSame('alt text here', $page->imgAlt()); - $this->assertSame('https://example.com/img.png', $page->imgSrc()); - } - - public function test_description_img_alt_and_img_src_are_empty_when_no_meta_matches(): void - { - $page = new Page('https://example.com', $this->html('No meta here', '')); - - $this->assertSame('', $page->description()); - $this->assertSame('', $page->imgAlt()); - $this->assertSame('', $page->imgSrc()); - $this->assertSame([''], $page->keywords()); - } - - public function test_keywords_merges_trims_and_splits_all_matching_meta_values(): void - { - $page = new Page('https://example.com', $this->html( - <<<'HTML' - - - - HTML, - '', - )); - - $this->assertSame( - ['tag1', 'tag2', 'tag3', 'tag3', 'tag4', 'tag5'], - $page->keywords(), - ); - } - - public function test_body_returns_joined_paragraph_text_from_article(): void - { - $page = new Page('https://example.com', $this->html('', <<<'HTML' -
-

First paragraph.

-

-

Second paragraph.

-
- HTML)); - - $this->assertSame('First paragraph. Second paragraph.', $page->body()); - } - - public function test_body_falls_back_to_body_tag_when_no_article_or_main(): void - { - $page = new Page('https://example.com', $this->html('', '

Only body paragraph.

')); - - $this->assertSame('Only body paragraph.', $page->body()); - } - - public function test_links_can_be_relative(): void - { - $page = new Page('https://www.example.com/article', $this->html('', <<<'HTML' - Internal with www - External - Empty href - Relative path - HTML)); - - $exp = [ - 'https://www.example.com/other-page', - 'https://www.example.com/article/relative/path', - ]; - $act = $page->links()->all(); - $this->assertSameSize($exp, $act); - for ($i = 0; $i < count($exp); $i++) { - $this->assertSame($exp[$i], $act[$i]); - } - } - - public function test_to_array_contains_all_expected_keys(): void - { - $page = new Page('https://www.example.com/article', $this->html( - <<<'HTML' - Article Title - - - - - HTML, - '

Body text.

', - )); - - $this->assertSame([ - 'body' => 'Body text.', - 'description' => 'Article description', - 'domain' => 'example.com', - 'img_alt' => 'Alt text', - 'img_url' => 'https://example.com/img.png', - 'keywords' => ['one', 'two'], - 'title' => 'Article Title', - 'url' => 'https://www.example.com/article', - ], $page->toArray()); - } -} From cf3ee0c78f8a327e01abd471a9fcb033605625c4 Mon Sep 17 00:00:00 2001 From: nelsw Date: Mon, 28 Sep 2026 19:18:13 -0400 Subject: [PATCH 2/2] removed remote logic to local engine --- app/Http/Controllers/SerpController.php | 36 +-- app/Services/GoService.php | 2 +- .../js/components/sitemaps/UrlTreeNode.vue | 251 +++++++++++++----- resources/js/pages/serps/Show.vue | 87 +++--- resources/js/pages/sitemaps/Show.vue | 48 +++- 5 files changed, 298 insertions(+), 126 deletions(-) diff --git a/app/Http/Controllers/SerpController.php b/app/Http/Controllers/SerpController.php index 2210204..dba1a9b 100644 --- a/app/Http/Controllers/SerpController.php +++ b/app/Http/Controllers/SerpController.php @@ -2,9 +2,9 @@ namespace App\Http\Controllers; -use App\Models\Page; use App\Models\Serp; use Illuminate\Http\RedirectResponse; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\URL; use Inertia\Inertia; use Inertia\Response; @@ -31,25 +31,31 @@ public function destroy(Serp $serp): RedirectResponse public function show(Serp $serp): Response { + $data = collect($serp->data ?? []); + return Inertia::render('serps/Show', [ 'serp' => [ 'id' => $serp->id, 'query' => $serp->query, - 'data' => $serp->data ?? [], - 'similarQueries' => $serp->data['similar_queries'] ?? [], + 'similarQueries' => collect($data->get('similar_queries', [])) + ->pluck('value') + ->filter() + ->values() + ->all(), 'screenshotUrl' => $serp->screenshotUrl(), - 'pages' => $serp->pages->map(fn (Page $page) => [ - 'id' => $page->id, - 'faviconUrl' => URL::toFavicon($page->url, 32), - 'kind' => str($page->kind)->replace('_', ' ')->title(), - 'index' => $page->index, - 'title' => $page->title, - 'url' => $page->url, - 'domain' => $page->domain, - 'created_at' => $page->created_at, - 'meta' => $page->meta ?? [], - 'screenshotUrl' => $page->screenshotUrl(), - ])->all(), + 'results' => $data->except('similar_queries') + ->flatMap(fn (array $items, string $key) => collect($items)->map(fn (array $item) => [ + 'id' => "$key-".($item['index'] ?? 0), + 'faviconUrl' => filled($item['url'] ?? null) ? URL::toFavicon($item['url'], 32) : null, + 'kind' => str($item['kind'] ?? $key)->replace('_', ' ')->title()->toString(), + 'index' => $item['index'] ?? null, + 'title' => $item['title'] ?? '', + 'url' => $item['url'] ?? null, + 'domain' => $item['domain'] ?? null, + 'meta' => Arr::except($item, ['kind', 'index', 'title', 'url', 'domain']), + ])) + ->values() + ->all(), ], ]); } diff --git a/app/Services/GoService.php b/app/Services/GoService.php index d88cea3..22c4e50 100644 --- a/app/Services/GoService.php +++ b/app/Services/GoService.php @@ -36,7 +36,7 @@ public function pub(Bot|int $bot): void { 'headless' => $bot->headless, 'query' => $bot->query, 'last_run_at' => $bot->lastRunAt(), - 'blacklist' => $bot->blacklist(), + 'blacklist' => $bot->blacklist, ])); } diff --git a/resources/js/components/sitemaps/UrlTreeNode.vue b/resources/js/components/sitemaps/UrlTreeNode.vue index 2e5daba..86acf0f 100644 --- a/resources/js/components/sitemaps/UrlTreeNode.vue +++ b/resources/js/components/sitemaps/UrlTreeNode.vue @@ -1,11 +1,7 @@