diff --git a/plugins/french/chireads.ts b/plugins/french/chireads.ts index 1d08feaa5..b872b476a 100644 --- a/plugins/french/chireads.ts +++ b/plugins/french/chireads.ts @@ -2,252 +2,311 @@ import { CheerioAPI, load } from 'cheerio'; import { fetchApi } from '@libs/fetch'; import { Plugin } from '@/types/plugin'; import { Filters, FilterTypes } from '@libs/filterInputs'; -import dayjs from 'dayjs'; import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; +type WordPressCategory = { + name: string; + link: string; +}; + class ChireadsPlugin implements Plugin.PluginBase { id = 'chireads'; name = 'Chireads'; icon = 'src/fr/chireads/icon.png'; site = 'https://chireads.com'; - version = '1.0.2'; + version = '2.3.4'; + + // The site is fronted by Cloudflare, which serves different HTML/JSON to a + // plain device User-Agent (the mobile app injects its own UA via fetchApi) + // than to a desktop browser. Send the same desktop Chrome UA on every + // request — HTML pages and the wp-json REST endpoints alike — so a novel's + // chapter list survives on the app. + private readonly browserHeaders = { + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36', + }; + + private readonly restHeaders = { + Accept: 'application/json, */*;q=0.8', + 'User-Agent': this.browserHeaders['User-Agent'], + }; async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); + const r = await fetchApi(url, { headers: this.browserHeaders }); + if (!r.ok) throw new Error(`HTTP ${r.status} while loading ${url}`); const body = await r.text(); - const $ = load(body); - return $; + return load(body); } - async popularNovels( - pageNo: number, - { filters, showLatestNovels }: Plugin.PopularNovelsOptions, - ): Promise { - let url = this.site; - let tag = 'all'; - if (showLatestNovels) url += '/category/translatedtales/page/' + pageNo; - else { - if ( - filters && - typeof filters.tag.value === 'string' && - filters.tag.value !== 'all' - ) - tag = filters.tag.value; - if (tag !== 'all') url += '/tag/' + tag + '/page/' + pageNo; - else if (pageNo > 1) return []; + private absoluteUrl(url?: string): string { + if (!url) return defaultCover; + try { + const absolute = new URL(url, this.site); + return /^https?:$/.test(absolute.protocol) ? absolute.href : defaultCover; + } catch { + return defaultCover; } - let $ = await this.getCheerio(url); + } + + private toPath(url?: string): string { + if (!url) return ''; + const parsed = new URL(url, this.site); + if (!/(?:^|\.)chireads\.com$/i.test(parsed.hostname)) return ''; + return `${parsed.pathname}${parsed.search}`; + } + + private compactChapterPath(url?: string): string { + if (!url) return ''; + const parsed = new URL(url, this.site); + if (!/(?:^|\.)chireads\.com$/i.test(parsed.hostname)) return ''; + const segments = parsed.pathname.split('/').filter(Boolean); + if (segments[0] === 'c' && segments[1]) return `/c/${segments[1]}/`; + const last = (offset: number) => segments[segments.length - offset]; + const hasDateSuffix = + /^\d{4}$/.test(last(3) || '') && + /^\d{1,2}$/.test(last(2) || '') && + /^\d{1,2}$/.test(last(1) || ''); + const chapterSlug = last(hasDateSuffix ? 4 : 1); + return chapterSlug ? `/c/${chapterSlug}/` : ''; + } + + private parseCards($: CheerioAPI): Plugin.NovelItem[] { const novels: Plugin.NovelItem[] = []; - let novel: Plugin.NovelItem; + const seen = new Set(); + $('ul.refresh-card-grid li.refresh-card').each((i, el) => { + const novelUrl = $(el).find('.refresh-card-title a').attr('href'); + if (!novelUrl || seen.has(novelUrl)) return; + seen.add(novelUrl); + novels.push({ + name: $(el).find('.refresh-card-title a').text().trim(), + cover: this.absoluteUrl( + $(el).find('.refresh-card-cover img').attr('src'), + ), + path: this.toPath(novelUrl), + }); + }); + return novels; + } - if (showLatestNovels || tag !== 'all') { - let loop = 1; - if (showLatestNovels) loop = 2; - for (let i = 0; i < loop; i++) { - if (i === 1) - $ = await this.getCheerio( - this.site + '/category/original/page/' + pageNo, - ); - let romans = $('.romans-content li'); - if (!romans.length) romans = $('#content li'); - romans.each((i, elem) => { - const novelName = $(elem) - .contents() - .find('div') + async popularNovels( + pageNo: number, + { filters, showLatestNovels }: Plugin.PopularNovelsOptions, + ): Promise { + if (showLatestNovels) { + if (pageNo !== 1) return []; + const $ = await this.getCheerio(this.site); + const novels: Plugin.NovelItem[] = []; + const seen = new Set(); + $('.dernieres-tabel tbody tr').each((i, el) => { + const novelUrl = $(el).find('td').first().find('a').attr('href'); + if (!novelUrl || seen.has(novelUrl)) return; + seen.add(novelUrl); + novels.push({ + name: $(el) + .find('td') .first() + .find('a') .text() - .trim(); - const novelCover = $(elem) - .find('div') - .first() - .find('img') - .attr('src'); - const novelUrl = $(elem).find('div').first().find('a').attr('href'); - - if (novelUrl) { - novel = { - name: novelName, - cover: novelCover, - path: novelUrl.replace(this.site, ''), - }; - novels.push(novel); - } + .trim() + .replace(/^\[[TO]\]\s*/, ''), + cover: defaultCover, + path: this.toPath(novelUrl), }); - } - } else { - const populaire = $(':contains("Populaire")') - .last() - .parent() - .next() - .find('li > div'); - if (populaire.length === 12) { - // pc - let novelCover: string | undefined; - let novelName: string | undefined; - let novelUrl: string | undefined; - populaire.each((i, elem) => { - if (i % 2 === 0) novelCover = $(elem).find('img').attr('src'); - else { - novelName = $(elem).text().trim(); - novelUrl = $(elem).find('a').attr('href'); + }); - if (!novelUrl) return; + // The homepage "latest" table carries no cover images, only links to + // each novel's page. Resolve covers from the detail pages so the list + // shows a real cover instead of the "not available" placeholder. + const covers = await Promise.allSettled( + novels.map(async novel => { + const page = await this.getCheerio(this.site + novel.path); + const cover = + page('.refresh-detail-cover img').attr('src') || + page('.refresh-detail-cover img').attr('data-src'); + return cover ? this.absoluteUrl(cover) : defaultCover; + }), + ); + return novels.map((novel, index) => ({ + ...novel, + cover: + covers[index]?.status === 'fulfilled' + ? covers[index].value + : defaultCover, + })); + } - novel = { - name: novelName, - cover: novelCover || defaultCover, - path: novelUrl.replace(this.site, ''), - }; + const tag = filters?.tag?.value; + const isAll = typeof tag !== 'string' || tag === '' || tag === 'all'; + const bases = isAll + ? ['/category/translatedtales', '/category/original'] + : [`/tag/${tag}`]; - novels.push(novel); - } - }); - } // mobile - else { - const imgs = populaire.find('div.popular-list-img img'); - const txts = populaire.find('div.popular-list-name'); - - txts.each((i, elem) => { - const novelName = $(elem).text().trim(); - const novelCover = $(imgs[i]).attr('src'); - const novelUrl = $(elem).find('a').attr('href'); + const catalogues = isAll + ? await Promise.allSettled( + bases.map(base => + this.getCheerio(`${this.site}${base}/page/${pageNo}`), + ), + ) + : [ + { + status: 'fulfilled' as const, + value: await this.getCheerio( + `${this.site}${bases[0]}/page/${pageNo}`, + ), + }, + ]; + if ( + isAll && + catalogues.every(catalogue => catalogue.status === 'rejected') + ) { + throw new Error('All catalogue pages failed'); + } - if (novelUrl) { - novel = { - name: novelName, - cover: novelCover, - path: novelUrl.replace(this.site, ''), - }; - novels.push(novel); - } - }); + const novels: Plugin.NovelItem[] = []; + const seen = new Set(); + for (const catalogue of catalogues) { + if (catalogue.status !== 'fulfilled') continue; + const $ = catalogue.value; + for (const novel of this.parseCards($)) { + if (seen.has(novel.path)) continue; + seen.add(novel.path); + novels.push(novel); } } return novels; } async parseNovel(novelPath: string): Promise { - const novel: Plugin.SourceNovel = { path: novelPath, name: 'Sans titre' }; + const novel: Plugin.SourceNovel = { + path: this.toPath(novelPath), + name: '', + }; - const $ = await this.getCheerio(this.site + novelPath); + const $ = await this.getCheerio(this.site + novel.path); - novel.name = - $('.inform-product-txt').first().text().trim() || - $('.inform-title').text().trim(); - novel.cover = - $('.inform-product img').attr('src') || - $('.inform-product-img img').attr('src') || - defaultCover; - novel.summary = - $('.inform-inform-txt').text().trim() || - $('.inform-intr-txt').text().trim(); + novel.name = $('h1.refresh-detail-title').first().text().trim(); + novel.cover = this.absoluteUrl( + $('.refresh-detail-cover img').attr('src') || + $('.refresh-detail-cover img').attr('data-src'), + ); + novel.summary = $('.refresh-detail-summary-content').text().trim(); - const infos = - $('div.inform-product-txt > div.inform-intr-col').text().trim() || - $('div.inform-inform-data > h6').text().trim(); - if (infos.includes('Auteur : ')) - novel.author = infos - .substring( - infos.indexOf('Auteur : ') + 9, - infos.indexOf('Statut de Parution : '), - ) - .trim(); - else if (infos.includes('Fantrad : ')) - novel.author = infos - .substring( - infos.indexOf('Fantrad : ') + 10, - infos.indexOf('Statut de Parution : '), - ) - .trim(); - else novel.author = 'Inconnu'; - switch ( - infos.substring(infos.indexOf('Statut de Parution : ') + 21).toLowerCase() - ) { - case 'en pause': - novel.status = NovelStatus.OnHiatus; - break; - case 'complet': - novel.status = NovelStatus.Completed; - break; - default: - novel.status = NovelStatus.Ongoing; - break; - } + $('.refresh-detail-meta > div').each((i, el) => { + const label = $(el).find('dt').text().trim(); + const value = $(el).find('dd').text().trim(); + if (label.includes('Auteur')) novel.author = value; + else if (label.includes('Statut')) { + const status = value.toLowerCase(); + if (status.includes('en pause') || status.includes('hiatus')) + novel.status = NovelStatus.OnHiatus; + else if (status.includes('complet') || status.includes('termin')) + novel.status = NovelStatus.Completed; + else novel.status = NovelStatus.Ongoing; + } + }); - const chapters: Plugin.ChapterItem[] = []; + const chapters = new Map(); + $('.refresh-detail-chapter-list a').each((i, el) => { + const chapterUrl = $(el).attr('href'); + const path = this.compactChapterPath(chapterUrl); + if (!path || chapters.has(path)) return; - let chapterList = $('.chapitre-table a'); - if (!chapterList.length) { - $('div.inform-annexe-list').first().remove(); - chapterList = $('.inform-annexe-list').find('a'); - } - chapterList.each((i, elem) => { - const chapterName = $(elem).text().trim(); - const chapterUrl = $(elem).attr('href'); - const releaseDate = dayjs( - chapterUrl?.substring(chapterUrl.length - 11, chapterUrl.length - 1), - ).format('DD MMMM YYYY'); + const title = $(el).text().trim(); + const match = title.match( + /^Chapitre\s+(\d+(?:[.,]\d+)?)\s*(?:(?:–|-|:)\s*)?(.*)$/i, + ); + const segments = new URL(chapterUrl!, this.site).pathname + .split('/') + .filter(Boolean); + const date = segments.slice(-3); + const hasDate = + /^\d{4}$/.test(date[0] || '') && + /^\d{1,2}$/.test(date[1] || '') && + /^\d{1,2}$/.test(date[2] || ''); - if (chapterUrl) { - chapters.push({ - name: chapterName, - releaseTime: releaseDate, - path: chapterUrl.replace(this.site, ''), - }); - } + chapters.set(path, { + name: match + ? `${match[1]}${match[2] ? ` - ${match[2].trim()}` : ''}` + : title, + path, + ...(match ? { chapterNumber: Number(match[1].replace(',', '.')) } : {}), + ...(hasDate + ? { + releaseTime: `${date[0]}-${date[1].padStart(2, '0')}-${date[2].padStart(2, '0')}`, + } + : {}), + }); }); - - novel.chapters = chapters; + novel.chapters = Array.from(chapters.values()); return novel; } async parseChapter(chapterUrl: string): Promise { - const $ = await this.getCheerio(this.site + chapterUrl); + const $ = await this.getCheerio( + this.site + this.compactChapterPath(chapterUrl), + ); - const chapterText = $('#content').html() || ''; + const content = $('#content').first(); + content + .find('script, style, iframe, form, nav, footer, .sharedaddy, .ads') + .remove(); + if (content.text().replace(/\s+/g, ' ').trim().length < 200) { + throw new Error('Chapter content is not readable'); + } - return chapterText; + return content.html() || ''; } async searchNovels( searchTerm: string, pageNo: number, ): Promise { - if (pageNo !== 1) return []; - let novels: Plugin.NovelItem[] = []; - - let i = 1; - let finised = false; - while (!finised) { - await this.popularNovels(i, { - showLatestNovels: true, - filters: undefined, - }).then(res => { - if (res.length === 0) finised = true; - novels.push(...res); - }); - i++; - } - - novels = novels.filter(novel => - novel.name - .toLowerCase() - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - .includes( - searchTerm - .toLowerCase() - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, ''), - ), + const parents = await Promise.allSettled( + [2, 811].map(async parent => { + const response = await fetchApi( + `${this.site}/wp-json/wp/v2/categories?parent=${parent}&search=${encodeURIComponent(searchTerm)}&per_page=100&page=${pageNo}`, + { headers: this.restHeaders }, + ); + if (!response.ok) throw new Error(`Search parent ${parent} failed`); + const categories: unknown = await response.json(); + if ( + !Array.isArray(categories) || + !categories.every( + category => + category !== null && + typeof category === 'object' && + typeof (category as WordPressCategory).name === 'string' && + typeof (category as WordPressCategory).link === 'string', + ) + ) + throw new Error(`Search parent ${parent} returned invalid data`); + return categories as WordPressCategory[]; + }), + ); + const categories = parents.flatMap(parent => + parent.status === 'fulfilled' ? parent.value : [], ); + if (parents.every(parent => parent.status === 'rejected')) + throw new Error('Chireads search failed for all category parents'); - return novels; + const seen = new Set(); + return categories + .map(category => ({ + name: category.name, + cover: defaultCover, + path: this.toPath(category.link), + })) + .filter( + novel => + (novel.path.startsWith('/category/translatedtales/') || + novel.path.startsWith('/category/original/')) && + !seen.has(novel.path) && + Boolean(seen.add(novel.path)), + ); } filters = { diff --git a/plugins/french/harkeneliwood.ts b/plugins/french/harkeneliwood.ts index 6631e3afc..946a76121 100644 --- a/plugins/french/harkeneliwood.ts +++ b/plugins/french/harkeneliwood.ts @@ -5,20 +5,46 @@ import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; import dayjs from 'dayjs'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class HarkenEliwoodPlugin implements Plugin.PluginBase { id = 'harkeneliwood'; name = 'HarkenEliwood'; icon = 'src/fr/harkeneliwood/icon.png'; site = 'https://harkeneliwood.wordpress.com'; - version = '1.0.0'; + version = '1.0.3'; async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); - const body = await r.text(); - const $ = load(body); - return $; + return load( + await fetchCheckedHtml(url, { + headers: { 'Accept-Encoding': 'deflate' }, + }), + ); } async popularNovels(pageNo: number): Promise { @@ -43,6 +69,14 @@ class HarkenEliwoodPlugin implements Plugin.PluginBase { novels.push(novel); } }); + await Promise.all( + novels.map(async item => { + const detail = await this.getCheerio(this.site + item.path); + item.cover = + detail('#content .entry-content p img').first().attr('src') || + defaultCover; + }), + ); return novels; } @@ -58,7 +92,7 @@ class HarkenEliwoodPlugin implements Plugin.PluginBase { $('#content .entry-content p img').first().attr('src') || defaultCover; novel.summary = this.getSummary($('#content .entry-content').text()); novel.author = this.getAuthor($('#content .entry-content').text()); - novel.status = NovelStatus.Ongoing; + novel.status = NovelStatus.Unknown; const chapters: Plugin.ChapterItem[] = []; $('#content .entry-content p a').each((i, elem) => { const chapterName = $(elem).text().trim(); @@ -67,7 +101,7 @@ class HarkenEliwoodPlugin implements Plugin.PluginBase { if (chapterUrl && chapterUrl.includes(this.site) && chapterName) { const releaseDate = dayjs( chapterUrl?.substring(this.site.length + 1, this.site.length + 11), - ).format('DD MMMM YYYY'); + ).format('YYYY-MM-DD'); chapters.push({ name: chapterName, path: chapterUrl.replace(this.site, ''), @@ -133,6 +167,7 @@ class HarkenEliwoodPlugin implements Plugin.PluginBase { const $ = await this.getCheerio(this.site + chapterPath); const title = $('h1.entry-title'); const chapter = $('div.entry-content'); + chapter.find('script, style, ins, iframe, .ads').remove(); return (title.html() || '') + (chapter.html() || ''); } diff --git a/plugins/french/jgarden.ts b/plugins/french/jgarden.ts new file mode 100644 index 000000000..27e5d8c50 --- /dev/null +++ b/plugins/french/jgarden.ts @@ -0,0 +1,330 @@ +import { load } from 'cheerio'; +import { defaultCover } from '@libs/defaultCover'; +import { fetchApi } from '@libs/fetch'; +import { NovelStatus } from '@libs/novelStatus'; +import { Plugin } from '@/types/plugin'; + +type WordPressPage = { + slug: string; + link: string; + title: { rendered: string }; + content: { rendered: string }; +}; + +const chapterSlug = + /(?:chapitre|prologue|epilogue|interlude|bonus|postface|preface)/i; + +class JGardenPlugin implements Plugin.PluginBase { + id = 'jgarden'; + name = 'J-Garden'; + icon = 'src/fr/jgarden/icon.png'; + site = 'https://j-garden.fr/'; + version = '1.0.5'; + + resolveUrl(path: string): string { + const url = new URL(path, this.site); + if (url.origin !== new URL(this.site).origin) + throw new Error('Cannot resolve a foreign origin'); + return url.toString(); + } + + private slugFromLink(link: string): string | undefined { + try { + const url = new URL(link, this.site); + if (url.origin !== new URL(this.site).origin) return undefined; + const parts = url.pathname.split('/').filter(Boolean); + return parts[parts.length - 1]; + } catch { + return undefined; + } + } + + private async getJson(path: string): Promise { + const response = await fetchApi(this.resolveUrl(path)); + if (!response.ok) throw new Error(`Failed to load ${path}`); + if (!/[/+]json\b/i.test(response.headers.get('content-type') || '')) + throw new Error(`Expected a JSON response for ${path}`); + return response.json() as Promise; + } + + private parseCatalogue(html: string): Plugin.NovelItem[] { + const $ = load(html); + const novels = new Map(); + $('a[href]').each((_, element) => { + const href = $(element).attr('href'); + const path = href ? this.slugFromLink(href) : undefined; + const image = $(element).find('img').first(); + const name = + $(element).text().trim() || + image.attr('alt')?.trim() || + this.nameFromSlug(path || ''); + const coverSrc = image.attr('src'); + const width = Number(image.attr('width')); + const height = Number(image.attr('height')); + // The catalogue renders wide series banners (e.g. 2567×487), not the + // portrait book covers. A landscape image zoomed into a portrait card + // looks broken, so fall back to the default cover instead. + const isBanner = + Number.isFinite(width) && Number.isFinite(height) && width > height; + const cover = + coverSrc && !isBanner ? this.resolveUrl(coverSrc) : defaultCover; + if (path && name) novels.set(path, { name, path, cover }); + }); + return Array.from(novels.values()); + } + + // The catalogue only carries wide series banners; the actual portrait book + // cover lives on each novel's own page. Fetch it there so the list shows a + // real cover instead of a zoomed banner or the fallback placeholder. + private async fetchCover(slug: string): Promise { + try { + const pages = await this.getJson<{ content: { rendered: string } }[]>( + `/wp-json/wp/v2/pages?slug=${encodeURIComponent(slug)}&_fields=content`, + ); + const content = pages[0]?.content?.rendered; + if (!content) return defaultCover; + const $ = load(content); + let cover = ''; + $('img').each((_, element) => { + if (cover) return; + const src = $(element).attr('src'); + if (!src) return; + const width = Number($(element).attr('width')); + const height = Number($(element).attr('height')); + const portrait = + !Number.isFinite(width) || + !Number.isFinite(height) || + height >= width; + if (portrait) cover = this.resolveUrl(src); + }); + return cover || defaultCover; + } catch { + return defaultCover; + } + } + + private nameFromSlug(slug: string): string { + return slug + .split('-') + .filter(Boolean) + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + } + + private chapterSequence(name: string, path: string) { + const source = `${name} ${path}`; + const volume = Number( + source.match(/(?:tome|volume|vol\.?|v|t)[\s_-]*(\d+)/i)?.[1] || 0, + ); + // Within each volume: preface, prologue, numbered chapters, interlude, + // bonus, epilogue, then postface. Unrecognised links retain DOM order. + const specialKinds = [ + ['preface', 0], + ['prologue', 1], + ['interlude', 3], + ['bonus', 4], + ['epilogue', 5], + ['postface', 6], + ] as const; + const special = specialKinds.find(([label]) => + new RegExp(`(?:^|[\\s_-])${label}(?:$|[\\s_-])`, 'i').test(source), + ); + const chapter = source.match( + /(?:chapitre|chapter|ch\.?)[\s_-]*(\d+(?:[.,]\d+)?)/i, + ); + return { + volume, + kind: special?.[1] ?? (chapter ? 2 : 7), + chapter: Number(chapter?.[1].replace(',', '.') || 0), + }; + } + + async popularNovels(pageNo: number): Promise { + if (pageNo > 1) return []; + const sections = await Promise.allSettled( + ['jg-ln', 'jg-web-novel'].map(async section => { + const pages = await this.getJson( + `/wp-json/wp/v2/pages?slug=${section}&_fields=content`, + ); + if ( + !Array.isArray(pages) || + !pages.every( + page => + page !== null && + typeof page === 'object' && + (page as Record).content !== null && + typeof (page as Record).content === 'object' && + typeof (page as { content: Record }).content + .rendered === 'string', + ) + ) + throw new Error(`Invalid catalogue section: ${section}`); + return pages as Pick[]; + }), + ); + const catalogues = sections.flatMap(section => + section.status === 'fulfilled' ? [section.value] : [], + ); + if (!catalogues.length) throw new Error('Failed to load catalogue'); + const novels = new Map(); + for (const section of catalogues.flat()) { + for (const novel of this.parseCatalogue(section.content.rendered)) { + novels.set(novel.path, novel); + } + } + const list = Array.from(novels.values()); + const covers = await Promise.allSettled( + list.map(novel => this.fetchCover(novel.path)), + ); + return list.map((novel, index) => ({ + ...novel, + cover: + covers[index]?.status === 'fulfilled' + ? covers[index].value + : defaultCover, + })); + } + + async parseNovel(novelPath: string): Promise { + const novelUrl = this.resolveUrl(novelPath); + const slug = this.slugFromLink(novelUrl) || novelPath; + const pages = await this.getJson( + `/wp-json/wp/v2/pages?slug=${encodeURIComponent(slug)}&_fields=slug,link,title,content`, + ); + const page = pages[0]; + if (!page) throw new Error('Novel not found'); + + const $ = load(page.content.rendered); + const chapters = new Map< + string, + { chapter: Plugin.ChapterItem; index: number } + >(); + const firstChapter = $('a[href]') + .filter((_, element) => { + const href = $(element).attr('href'); + const chapterPath = href ? this.slugFromLink(href) : undefined; + return Boolean(chapterPath && chapterSlug.test(chapterPath)); + }) + .first(); + + $('a[href]').each((index, element) => { + const href = $(element).attr('href'); + const path = href ? this.slugFromLink(href) : undefined; + const name = $(element).text().trim(); + if (path && name && chapterSlug.test(path) && !chapters.has(path)) { + chapters.set(path, { chapter: { name, path }, index }); + } + }); + + const details = $('body *') + .map((_, element) => $(element).text().trim()) + .get() + .filter(Boolean) + .join(' '); + const status = + /\b(?:termin(?:é|ée|e)|completed|complete|fini)(?![A-Za-zÀ-ÖØ-öø-ÿ0-9_])/i.test( + details, + ) + ? NovelStatus.Completed + : /\b(?:hiatus|en pause|pause)\b/i.test(details) + ? NovelStatus.OnHiatus + : /\b(?:en cours|ongoing|publication)\b/i.test(details) + ? NovelStatus.Ongoing + : NovelStatus.Unknown; + + return { + path: page.slug, + name: load(page.title.rendered).text().trim(), + cover: $('img').first().attr('src') + ? this.resolveUrl($('img').first().attr('src')!) + : defaultCover, + summary: firstChapter.prevAll().text().trim(), + status, + chapters: (() => { + const items = Array.from(chapters.values()).map(entry => ({ + ...entry, + sequence: this.chapterSequence( + entry.chapter.name, + entry.chapter.path, + ), + })); + // When only some chapters carry a volume marker (e.g. v1/v2 books + // alongside untagged side stories), the site's DOM order is the + // reading order; keep it instead of interleaving by chapter number. + const mixedVolumes = + items.some(item => item.sequence.volume > 0) && + items.some(item => item.sequence.volume === 0); + items.sort((left, right) => { + if (mixedVolumes) return left.index - right.index; + return ( + left.sequence.volume - right.sequence.volume || + left.sequence.kind - right.sequence.kind || + left.sequence.chapter - right.sequence.chapter || + left.index - right.index + ); + }); + return items.map(({ chapter }, index) => ({ + ...chapter, + chapterNumber: index + 1, + })); + })(), + }; + } + + async parseChapter(chapterPath: string): Promise { + const chapterUrl = this.resolveUrl(chapterPath); + const slug = this.slugFromLink(chapterUrl) || chapterPath; + let posts = await this.getJson< + Pick[] + >( + `/wp-json/wp/v2/posts?slug=${encodeURIComponent(slug)}&_fields=content,title,link`, + ); + if (posts.length === 0) { + const resolved = await fetchApi(this.resolveUrl(chapterPath)); + const canonicalSlug = this.slugFromLink(resolved.url); + if (canonicalSlug && canonicalSlug !== slug) { + posts = await this.getJson( + `/wp-json/wp/v2/posts?slug=${encodeURIComponent(canonicalSlug)}&_fields=content,title,link`, + ); + } + } + const post = posts[0]; + if (!post) throw new Error('Chapter not found'); + + const $ = load(post.content.rendered); + const content = $('.elementor-widget-theme-post-content').first(); + const chapter = content.length ? content : $('body'); + chapter + .find( + 'script, style, nav, .sharedaddy, .share, [class*="share"], [id*="share"]', + ) + .remove(); + chapter + .find('*') + .filter((_, element) => !$(element).text().trim()) + .remove(); + const html = chapter.html()?.trim() || ''; + if (chapter.text().trim().length < 200) + throw new Error('No readable chapter content found'); + return html; + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + if (pageNo !== 1) return []; + const normalize = (value: string) => + value + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .trim(); + const query = normalize(searchTerm); + return (await this.popularNovels(1)).filter(novel => + normalize(novel.name).includes(query), + ); + } +} + +export default new JGardenPlugin(); diff --git a/plugins/french/kisswood.ts b/plugins/french/kisswood.ts index 7d18c2abf..d0e197b7e 100644 --- a/plugins/french/kisswood.ts +++ b/plugins/french/kisswood.ts @@ -4,18 +4,65 @@ import { Plugin } from '@/types/plugin'; import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class KissWoodPlugin implements Plugin.PluginBase { id = 'kisswood'; name = 'KissWood'; icon = 'src/fr/kisswood/icon.png'; site = 'https://kisswood.eu'; - version = '1.0.0'; + version = '1.0.3'; + + private async findMovedChapter(chapterPath: string): Promise { + const slug = chapterPath.split('/').filter(Boolean).pop() || ''; + const match = slug.match(/^(.*?)-chapitre-(\d+)/i); + if (!match) return null; + const query = `${match[1].replace(/-/g, ' ')} chapitre ${match[2]}`; + const response = await fetchApi( + `${this.site}/wp-json/wp/v2/search?search=${encodeURIComponent(query)}&type=post&subtype=post&per_page=20`, + ); + if (!response.ok) return null; + const results = (await response.json()) as { + title?: string; + url?: string; + }[]; + const replacement = results.find(result => + new RegExp(`chapitre\\D*${match[2]}(?:\\D|$)`, 'i').test( + result.title || '', + ), + )?.url; + if (!replacement) return null; + const url = new URL(replacement, this.site); + return url.origin === new URL(this.site).origin ? url.toString() : null; + } async getCheerio(url: string): Promise { - const r = await fetchApi(url); - const body = await r.text(); - const $ = load(body); - return $; + return load(await fetchCheckedHtml(url)); } async getNovelsCovers( @@ -126,7 +173,7 @@ class KissWoodPlugin implements Plugin.PluginBase { let novel: Plugin.SourceNovel = { path: novelPath, name: 'Sans titre', - status: NovelStatus.Ongoing, + status: NovelStatus.Unknown, }; const $ = await this.getCheerio(this.site + novelPath); @@ -153,6 +200,7 @@ class KissWoodPlugin implements Plugin.PluginBase { ].join(', '); const chapters: Plugin.ChapterItem[] = []; + const chapterPaths = new Set(); $(chapterSelectors).each((i, elem) => { const chapterName = $(elem).text().trim(); const chapterUrl = $(elem).attr('href')?.replace('http://', 'https://'); @@ -164,13 +212,14 @@ class KissWoodPlugin implements Plugin.PluginBase { !chapterUrl.includes('share=facebook') && !chapterUrl.includes('share=x') && !chapterUrl.includes('/category/traductions/') && - !chapterUrl.includes('/category/tour-des-mondes/') && - // Removal of duplicates - !chapters.some(chapter => this.site + chapter.path === chapterUrl) + !chapterUrl.includes('/category/tour-des-mondes/') ) { + const path = chapterUrl.replace(this.site, ''); + if (chapterPaths.has(path)) return; + chapterPaths.add(path); chapters.push({ name: chapterName, - path: chapterUrl.replace(this.site, ''), + path, }); } }); @@ -179,35 +228,58 @@ class KissWoodPlugin implements Plugin.PluginBase { } async parseChapter(chapterPath: string): Promise { - const $ = await this.getCheerio(this.site + chapterPath); + let body: string; + try { + body = await fetchCheckedHtml(this.site + chapterPath); + } catch (error) { + const replacement = await this.findMovedChapter(chapterPath); + if (!replacement) throw error; + body = await fetchCheckedHtml(replacement); + } - const elements: string[] = $('.entry-content') + const $ = load(body); + const chapter = $('.entry-content').first().clone(); + chapter + .find( + 'script, style, ins, iframe, .ads, .sharedaddy, [class*="sharing"], [id*="sharing"]', + ) + .remove(); + + const elements = chapter .contents() - .map((_, el) => $.html(el)) + .map((_, element) => $.html(element)) .get(); - - let hrIndexes: number[] = elements - .map((elem, index) => (elem.includes('
') ? index : -1)) + const separators = elements + .map((element, index) => (/ index !== -1); - - if (hrIndexes.length === 0) { - hrIndexes = [ - 0, - elements.findIndex( - element => - element.includes('https://fr.tipeee.com/kisswood/') || - element.includes('>Sommaire') || - element.includes('>Chapitre Suivant') || - element.includes('———————————————————————————-') || - element.includes('share=facebook'), - ), - ]; - } else if (hrIndexes.length === 1) { - hrIndexes.unshift(0); - } else { - hrIndexes[0] += 1; - } - return elements.slice(hrIndexes[0], hrIndexes[1]).join('\n'); + const markerIndex = elements.findIndex( + element => + !/Sommaire', + '>Chapitre Suivant', + '———————————————————————————-', + 'share=facebook', + ].some(marker => element.includes(marker)), + ); + const start = separators.length > 1 ? separators[0] + 1 : 0; + const end = + separators.length > 1 + ? separators[1] + : separators.length === 1 + ? separators[0] + : markerIndex >= 0 + ? markerIndex + : elements.length; + const content = elements.slice(start, end).join('\n'); + const parsedContent = load(content); + if ( + parsedContent.text().replace(/\s+/g, ' ').trim().length < 200 && + !parsedContent('img').length + ) + throw new Error('No readable chapter content found'); + return content; } async searchNovels( diff --git a/plugins/french/lightnovelvf.ts b/plugins/french/lightnovelvf.ts new file mode 100644 index 000000000..2ecc8c20a --- /dev/null +++ b/plugins/french/lightnovelvf.ts @@ -0,0 +1,323 @@ +import { load } from 'cheerio'; +import { defaultCover } from '@libs/defaultCover'; +import { fetchApi } from '@libs/fetch'; +import { NovelStatus } from '@libs/novelStatus'; +import { Plugin } from '@/types/plugin'; + +type ChapterEntry = { + number: string; + slug: string; + name_fr?: string | null; + name?: string | null; + created_at?: string | null; +}; + +type ChapterPage = { + chapters: ChapterEntry[]; + current_page: number; + last_page: number; + total: number; +}; + +class LightNovelVFPlugin implements Plugin.PagePlugin { + id = 'lightnovelvf'; + name = 'LightNovelVF'; + icon = 'src/fr/lightnovelvf/icon.png'; + site = 'https://www.lightnovelvf.com/'; + version = '1.0.4'; + + resolveUrl(path: string, _isNovel = false): string { + void _isNovel; + const url = new URL(path, this.site); + if (url.origin !== new URL(this.site).origin) + throw new Error('Cannot resolve a foreign origin'); + const cleanPath = url.pathname + .replace(/^\/+|\/+$/g, '') + .replace(/^novel\//, ''); + return new URL(`/novel/${cleanPath}`, this.site).toString(); + } + + private async fetchHtml(url: string): Promise { + const response = await fetchApi(url); + if (!response.ok) throw new Error(`Failed to load ${url}`); + return response.text(); + } + + private retryDelay(response: Response | null, attempt: number): number { + const retryAfter = response?.headers.get('retry-after'); + if (retryAfter) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000; + const date = Date.parse(retryAfter); + if (!Number.isNaN(date)) return Math.max(date - Date.now(), 0); + } + return 1000 * 2 ** attempt; + } + + private isChapterPageMetadata( + value: unknown, + requestedPage: number, + ): value is Omit & { chapters: unknown[] } { + if (!value || typeof value !== 'object') return false; + const page = value as Record; + return ( + Array.isArray(page.chapters) && + typeof page.current_page === 'number' && + Number.isInteger(page.current_page) && + page.current_page > 0 && + page.current_page === requestedPage && + typeof page.last_page === 'number' && + Number.isInteger(page.last_page) && + page.last_page >= page.current_page && + typeof page.total === 'number' && + Number.isInteger(page.total) && + page.total >= 0 + ); + } + + private isChapterEntry(value: unknown): value is ChapterEntry { + if (!value || typeof value !== 'object') return false; + const chapter = value as Record; + return ( + typeof chapter.number === 'string' && + chapter.number.trim().length > 0 && + Number.isFinite(Number(chapter.number)) && + typeof chapter.slug === 'string' && + chapter.slug.trim().length > 0 && + (chapter.name_fr === undefined || + chapter.name_fr === null || + typeof chapter.name_fr === 'string') && + (chapter.name === undefined || + chapter.name === null || + typeof chapter.name === 'string') && + (chapter.created_at === undefined || + chapter.created_at === null || + typeof chapter.created_at === 'string') + ); + } + + private async fetchChapterPage( + url: string, + requestedPage: number, + ): Promise { + for (let attempt = 0; attempt < 6; attempt += 1) { + let response: Response; + try { + response = await fetchApi(url); + } catch { + if (attempt === 5) throw new Error(`Failed to load ${url}`); + await new Promise(resolve => + setTimeout(resolve, this.retryDelay(null, attempt)), + ); + continue; + } + if ( + (response.status === 429 || + (response.status >= 500 && response.status < 600)) && + attempt < 5 + ) { + await new Promise(resolve => + setTimeout(resolve, this.retryDelay(response, attempt)), + ); + continue; + } + if (!response.ok) throw new Error(`Failed to load ${url}`); + if (!response.headers.get('content-type')?.includes('application/json')) + throw new Error(`Expected JSON chapter page from ${url}`); + const page: unknown = await response.json(); + if (!this.isChapterPageMetadata(page, requestedPage)) + throw new Error(`Invalid chapter page from ${url}`); + if (!page.chapters.every(chapter => this.isChapterEntry(chapter))) + throw new Error(`Invalid chapter entry from ${url}`); + return page as ChapterPage; + } + throw new Error(`Failed to load ${url}`); + } + + private parseCards(html: string): Plugin.NovelItem[] { + const $ = load(html); + const novels = new Map(); + $('a[href^="/novel/"]').each((_, element) => { + const href = $(element).attr('href') || ''; + const match = href.match(/^\/novel\/([^/?#]+)\/?$/); + if (!match) return; + + const card = $(element).clone(); + card.find('img').remove(); + card + .find('*') + .filter((_, child) => { + const text = $(child).text().trim(); + return ( + /\b\d[\d\s,.]*\s*(?:chapitres?|chapters?|ch\.?)(?:\s|$)/i.test( + text, + ) || + /^(?:note|rating\s*:?)?\s*\d(?:[.,]\d+)?\s*(?:\/\s*5)?$/i.test(text) + ); + }) + .remove(); + const name = card.text().replace(/\s+/g, ' ').trim(); + if (!name) return; + + const cover = + $(element).find('img').first().attr('data-src') || + $(element).find('img').first().attr('src'); + novels.set(match[1], { + name, + path: match[1], + cover: cover ? new URL(cover, this.site).toString() : defaultCover, + }); + }); + return Array.from(novels.values()); + } + + private catalogueUrl( + pageNo: number, + searchTerm?: string, + latest = false, + ): string { + const parts = [`page=${pageNo}`]; + if (searchTerm) parts.push(`search=${encodeURIComponent(searchTerm)}`); + if (latest) parts.push('sort=update', 'sort_dir=desc'); + return new URL(`/novels-list?${parts.join('&')}`, this.site).toString(); + } + + private labelledValue(html: string, label: string): string | undefined { + const $ = load(html); + let value: string | undefined; + $('dt, [class*="label" i]').each((_, element) => { + if (value || $(element).text().trim().toLowerCase() !== label) return; + value = $(element).next('dd').first().text().trim() || undefined; + }); + return value; + } + + async popularNovels( + pageNo: number, + { showLatestNovels }: Plugin.PopularNovelsOptions, + ): Promise { + return this.parseCards( + await this.fetchHtml( + this.catalogueUrl(pageNo, undefined, Boolean(showLatestNovels)), + ), + ); + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + const normalizedSearchTerm = searchTerm.replace(/\\(?=%)/g, ''); + return this.parseCards( + await this.fetchHtml(this.catalogueUrl(pageNo, normalizedSearchTerm)), + ); + } + + private chapterItems(page: ChapterPage, slug: string): Plugin.ChapterItem[] { + const chapters = new Map(); + for (const chapter of page.chapters) { + const chapterNumber = Number(chapter.number); + if (!chapter.slug || !Number.isFinite(chapterNumber)) continue; + chapters.set(chapter.slug, { + name: chapter.name_fr || chapter.name || `Chapitre ${chapter.number}`, + path: `${slug}/${chapter.slug}`, + chapterNumber, + releaseTime: chapter.created_at || null, + }); + } + return Array.from(chapters.values()).sort( + (left, right) => (left.chapterNumber || 0) - (right.chapterNumber || 0), + ); + } + + private chapterPageUrl(slug: string, pageNo: number): string { + return `${this.resolveUrl(`${slug}/chapitres`, true)}?p=${pageNo}&order=asc&q=`; + } + + async parseNovel( + novelPath: string, + ): Promise { + const novelUrl = this.resolveUrl(novelPath, true); + const slug = new URL(novelUrl).pathname.replace(/^\/novel\//, ''); + const html = await this.fetchHtml(novelUrl); + const $ = load(html); + const statusText = + this.labelledValue(html, 'statut') || + $('span') + .filter((_, element) => + /^(?:en\s+cours|terminé|complete|hiatus|pause)$/i.test( + $(element).text().trim(), + ), + ) + .first() + .text() + .trim(); + const firstPage = await this.fetchChapterPage( + this.chapterPageUrl(slug, 1), + 1, + ); + + const cover = + $('.lnv-novel-cover, .lnv-novel__cover, .lnv-hero img, .hero img') + .first() + .attr('src') || $('img').first().attr('src'); + return { + path: slug, + name: $('h1').first().text().trim(), + cover: cover ? new URL(cover, this.site).toString() : defaultCover, + summary: $('.lnv-synopsis__body').first().text().trim() || undefined, + author: + this.labelledValue(html, 'auteur') || + $('[itemprop="author"]').first().text().trim() || + undefined, + genres: + this.labelledValue(html, 'catégories') || + this.labelledValue(html, 'categories') || + $('[itemprop="genre"]') + .map((_, element) => $(element).text().trim()) + .get() + .filter(Boolean) + .join(', ') || + undefined, + status: /termin|complet/i.test(statusText) + ? NovelStatus.Completed + : /hiatus|pause/i.test(statusText) + ? NovelStatus.OnHiatus + : /cours|ongoing/i.test(statusText) + ? NovelStatus.Ongoing + : NovelStatus.Unknown, + chapters: this.chapterItems(firstPage, slug), + totalPages: Math.max(firstPage.last_page, 1), + }; + } + + async parsePage(novelPath: string, page: string): Promise { + const pageNo = Number(page); + if (!Number.isInteger(pageNo) || pageNo < 1) + throw new Error('Invalid page'); + const novelUrl = this.resolveUrl(novelPath, true); + const slug = new URL(novelUrl).pathname.replace(/^\/novel\//, ''); + const pageData = await this.fetchChapterPage( + this.chapterPageUrl(slug, pageNo), + pageNo, + ); + return { chapters: this.chapterItems(pageData, slug) }; + } + + async parseChapter(chapterPath: string): Promise { + const html = await this.fetchHtml(this.resolveUrl(chapterPath)); + const $ = load(html); + const content = $('.lnv-reader-content').first(); + content + .find( + 'script, style, header, footer, nav, form, [class*="nav" i], [id*="nav" i], [class*="advert" i], [id*="advert" i], [class*="share" i], [class*="control" i]', + ) + .remove(); + const chapter = content.html()?.trim() || ''; + if (content.text().replace(/\s+/g, ' ').trim().length < 200) + throw new Error('No readable chapter content found'); + return chapter; + } +} + +export default new LightNovelVFPlugin(); diff --git a/plugins/french/noveldeglace.ts b/plugins/french/noveldeglace.ts index 10410df1c..ea74bfdf0 100644 --- a/plugins/french/noveldeglace.ts +++ b/plugins/french/noveldeglace.ts @@ -5,21 +5,46 @@ import { NovelStatus } from '@libs/novelStatus'; import { Filters, FilterTypes } from '@libs/filterInputs'; import { defaultCover } from '@libs/defaultCover'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class NovelDeGlacePlugin implements Plugin.PluginBase { id = 'noveldeglace'; name = 'NovelDeGlace'; icon = 'src/fr/noveldeglace/icon.png'; site = 'https://noveldeglace.com/'; - version = '1.0.5'; + version = '1.0.8'; - async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); - if (!r.ok) return undefined; - const body = await r.text(); - const loadedCheerio = load(body); - return loadedCheerio; + async getCheerio(url: string): Promise { + return load( + await fetchCheckedHtml(url, { + headers: { 'Accept-Encoding': 'deflate' }, + }), + ); } parseNovels( @@ -76,14 +101,11 @@ class NovelDeGlacePlugin implements Plugin.PluginBase { } url += '/page/' + pageNo; const $ = await this.getCheerio(url); - if (!$) return []; return this.parseNovels($, showLatestNovels); } async parseNovel(novelPath: string): Promise { const $ = await this.getCheerio(this.site + novelPath); - if (!$) throw new Error('Failed to load page (open in web view)'); - const novel: Plugin.SourceNovel = { path: novelPath, name: 'Untitled' }; novel.name = $('span.current').text().trim(); @@ -214,19 +236,26 @@ class NovelDeGlacePlugin implements Plugin.PluginBase { }); }); - novel.chapters = novelChapters; + novel.chapters = novelChapters.map((chapter, index) => ({ + ...chapter, + chapterNumber: index, + })); return novel; } async parseChapter(chapterPath: string): Promise { const $ = await this.getCheerio(this.site + chapterPath); - if (!$) throw new Error('Failed to load page (open in web view)'); - - $('.mistape_caption').remove(); - const chapterText = - $('.chapter-content').html() || $('.entry-content').html() || ''; - return chapterText; + const chapter = $('.chapter-content').first().length + ? $('.chapter-content').first() + : $('.entry-content').first(); + chapter.find('script, style, ins, iframe, .ads, .mistape_caption').remove(); + if ( + chapter.text().replace(/\s+/g, ' ').trim().length < 200 && + !chapter.find('img').length + ) + throw new Error('No readable chapter content found'); + return chapter.html() || ''; } async searchNovels( @@ -236,8 +265,6 @@ class NovelDeGlacePlugin implements Plugin.PluginBase { if (num !== 1) return []; // only 1 page of results const url = this.site + 'roman'; const $ = await this.getCheerio(url); - if (!$) throw new Error('Failed to load page (open in web view)'); - let novels = this.parseNovels($, false); novels = novels.filter(novel => diff --git a/plugins/french/novhell.ts b/plugins/french/novhell.ts index 2253c7442..a71ea2013 100644 --- a/plugins/french/novhell.ts +++ b/plugins/french/novhell.ts @@ -4,20 +4,46 @@ import { Plugin } from '@/types/plugin'; import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class NovhellPlugin implements Plugin.PluginBase { id = 'novhell'; name = 'Novhell'; icon = 'src/fr/novhell/icon.png'; site = 'https://novhell.org'; - version = '1.0.1'; + version = '1.0.4'; async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); - const body = await r.text(); - const $ = load(body); - return $; + return load( + await fetchCheckedHtml(url, { + headers: { 'Accept-Encoding': 'deflate' }, + }), + ); } async popularNovels(pageNo: number): Promise { @@ -65,7 +91,7 @@ class NovhellPlugin implements Plugin.PluginBase { ?.replace('- NovHell', '') || ''; novel.cover = $('section div div div div div img').first().attr('src') || defaultCover; - novel.status = NovelStatus.Ongoing; + novel.status = NovelStatus.Unknown; novel.author = $("strong:contains('Ecrit par ')") .parent() .text() @@ -107,6 +133,7 @@ class NovhellPlugin implements Plugin.PluginBase { .replace(':', '') .trim(); const chapters: Plugin.ChapterItem[] = []; + const chapterPaths = new Set(); $('main div article div div section div div div div div p a').each( (i, elem) => { @@ -118,6 +145,9 @@ class NovhellPlugin implements Plugin.PluginBase { const chapterUrl = $(elem).attr('href'); // Check if the chapter URL exists and contains the site name. if (chapterUrl && chapterUrl.includes(this.site)) { + const path = chapterUrl.replace(this.site, ''); + if (chapterPaths.has(path)) return; + chapterPaths.add(path); const regex = /Chapitre (\d+)/g; let chapterNumber = 0; let match; @@ -127,7 +157,7 @@ class NovhellPlugin implements Plugin.PluginBase { } chapters.push({ name: chapterName, - path: chapterUrl.replace(this.site, ''), + path, chapterNumber: chapterNumber, }); } @@ -170,10 +200,16 @@ class NovhellPlugin implements Plugin.PluginBase { const chapter = sections.eq(numberOfSection - positionChapter); if (title && chapter) { - return (title.html() || '') + (chapter.html() || ''); + const content = (title.html() || '') + (chapter.html() || ''); + const parsedContent = load(content); + if ( + parsedContent.text().replace(/\s+/g, ' ').trim().length >= 200 || + parsedContent('img').length + ) + return content; } } - return ''; + throw new Error('No readable chapter content found'); } async searchNovels( diff --git a/plugins/french/phenixscans.broken.ts b/plugins/french/phenixscans.broken.ts deleted file mode 100644 index ebb7421b4..000000000 --- a/plugins/french/phenixscans.broken.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { CheerioAPI, load } from 'cheerio'; -import { fetchApi } from '@libs/fetch'; -import { Plugin } from '@/types/plugin'; -import { Filters, FilterTypes } from '@libs/filterInputs'; -import { defaultCover } from '@libs/defaultCover'; -import { NovelStatus } from '@libs/novelStatus'; -import dayjs from 'dayjs'; - -class PhenixScansTradPlugin implements Plugin.PluginBase { - id = 'phenixscans'; - name = 'PhenixScans'; - icon = 'src/fr/phenixscans/icon.png'; - site = 'https://phenixscans.fr'; - version = '1.0.1'; - - async getCheerio(url: string): Promise { - const r = await fetchApi(url); - const body = await r.text(); - const $ = load(body); - return $; - } - - async popularNovels( - pageNo: number, - { showLatestNovels, filters }: Plugin.PopularNovelsOptions, - ): Promise { - const novels: Plugin.NovelItem[] = []; - let novel: Plugin.NovelItem; - - let filter = ''; - for (const key in filters) { - if (typeof filters[key].value === 'object') { - for (const value of filters[key].value as string[]) { - filter += `&genre%5B%5D=${value}`; - } - } - } - - const order = showLatestNovels ? 'update' : 'popular'; - const url = `${this.site}/manga/?page=${pageNo}${filter}&status=&type=novel&order=${order}`; - - const $ = await this.getCheerio(url); - $('div div div div a').each((i, elem) => { - const novelName = $(elem).attr('title')?.trim(); - const novelUrl = $(elem).attr('href'); - const novelCover = $(elem).find('div img').attr('src') || defaultCover; - - if (novelUrl && novelName) { - novel = { - name: novelName, - cover: novelCover, - path: novelUrl.replace(this.site, ''), - }; - novels.push(novel); - } - }); - return novels; - } - - async parseNovel(novelPath: string): Promise { - const novel: Plugin.SourceNovel = { - path: novelPath, - name: 'Sans titre', - }; - - const $ = await this.getCheerio(this.site + novelPath); - - novel.name = $('h1[itemprop=name]').text().replace('– Novel', '').trim(); - novel.cover = - $('div[itemprop=image] img').first().attr('src') || defaultCover; - novel.author = $('.fmed b:contains(Auteur)+span').text().trim(); - novel.genres = $('.mgen a') - .map(function () { - return $(this).text(); - }) - .get() - .join(', '); - novel.summary = $('.entry-content[itemprop=description]').text().trim(); - novel.status = this.getStatus( - $('.tsinfo .imptdt:contains(Statut)').text().replace('Statut', '').trim(), - ); - - const chapters: Plugin.ChapterItem[] = []; - $('ul li:has(div.chbox):has(div.eph-num)').each((i, elem) => { - const chapterName = $(elem).find('a .chapternum').text().trim(); - const chapterUrl = $(elem).find('a').attr('href'); - const releaseDate = this.parseDate($(elem).find('a .chapterdate').text()); - if (chapterUrl && chapterUrl.includes(this.site) && chapterName) { - chapters.push({ - name: chapterName, - path: chapterUrl.replace(this.site, ''), - releaseTime: releaseDate, - }); - } - }); - novel.chapters = chapters; - return novel; - } - - parseDate(date: string): string { - const monthMapping: Record = { - janvier: 1, - fevrier: 2, - mars: 3, - avril: 4, - mai: 5, - juin: 6, - juillet: 7, - aout: 8, - septembre: 9, - octobre: 10, - novembre: 11, - decembre: 12, - }; - - const [day, month, year] = date.split(' '); - return dayjs( - `${day} ${monthMapping[month.normalize('NFD').replace(/[\u0300-\u036f]/g, '')]} ${year}`, - 'D MMMM YYYY', - ).format('DD MMMM YYYY'); - } - - getStatus(status: string) { - const lowerCaseStatus = status.toLowerCase(); - const ongoing = ['en cours', 'en cours de publication']; - const onhiatus = ['en pause', 'en attente']; - const completed = ['complété', 'fini', 'achevé', 'terminé']; - const cancelled = ['abandonné']; - - if (ongoing.includes(lowerCaseStatus)) { - return NovelStatus.Ongoing; - } else if (onhiatus.includes(lowerCaseStatus)) { - return NovelStatus.OnHiatus; - } else if (completed.includes(lowerCaseStatus)) { - return NovelStatus.Completed; - } else if (cancelled.includes(lowerCaseStatus)) { - return NovelStatus.Cancelled; - } - return NovelStatus.Unknown; - } - - async parseChapter(chapterPath: string): Promise { - const $ = await this.getCheerio(this.site + chapterPath); - return $('#readerarea').html() || ''; - } - - async searchNovels( - searchTerm: string, - pageNo: number, - ): Promise { - if (pageNo !== 1) return []; - - const popularNovels = this.popularNovels(1, { - showLatestNovels: true, - filters: undefined, - }); - - const novels = (await popularNovels).filter(novel => - novel.name - .toLowerCase() - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - .trim() - .includes( - searchTerm - .toLowerCase() - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - .trim(), - ), - ); - - return novels; - } - - filters = { - genre: { - type: FilterTypes.CheckboxGroup, - label: 'Genre', - value: [], - options: [ - { label: 'Action', value: 'action' }, - { - label: 'Action Adventure Fantaisie Psychologique', - value: 'action-adventure-fantaisie-psychologique', - }, - { - label: 'Action Arts martiaux Aventure Fantastique Surnaturel', - value: 'action-arts-martiaux-aventure-fantastique-surnaturel', - }, - { - label: 'Action Aventure Fantastique', - value: 'action-aventure-fantastique', - }, - { label: 'Action Drame Shônen', value: 'action-drame-shonen' }, - { label: 'Adult', value: 'adult' }, - { label: 'Adventure', value: 'adventure' }, - { label: 'amitié', value: 'amitie' }, - { label: 'amour', value: 'amour' }, - { label: 'Art-martiaux', value: 'art-martiaux' }, - { label: 'Arts-martiaux', value: 'arts-martiaux' }, - { label: 'Aventure', value: 'aventure' }, - { label: 'Combat', value: 'combat' }, - { label: 'Comedie', value: 'comedie' }, - { label: 'Comedy', value: 'comedy' }, - { label: 'Demons', value: 'demons' }, - { label: 'Dragon', value: 'dragon' }, - { label: 'Drama', value: 'drama' }, - { label: 'Drame', value: 'drame' }, - { label: 'Ecchi', value: 'ecchi' }, - { label: 'Fantaisie', value: 'fantaisie' }, - { label: 'fantastique', value: 'fantastique' }, - { label: 'Fantasy', value: 'fantasy' }, - { label: 'Ghosts', value: 'ghosts' }, - { label: 'Harem', value: 'harem' }, - { label: 'Historical', value: 'historical' }, - { label: 'Historique', value: 'historique' }, - { label: 'Horreur', value: 'horreur' }, - { label: 'Horror', value: 'horror' }, - { label: 'Isekai', value: 'isekai' }, - { label: 'Josei', value: 'josei' }, - { label: 'longstrip art', value: 'longstrip-art' }, - { label: 'Magic', value: 'magic' }, - { label: 'Magie', value: 'magie' }, - { label: 'Male Protagonist', value: 'male-protagonist' }, - { label: 'manga', value: 'manga' }, - { label: 'Manhwa', value: 'manhwa' }, - { label: 'Manhwa Player', value: 'manhwa-player' }, - { label: 'Manwha', value: 'manwha' }, - { label: 'Martial Arts', value: 'martial-arts' }, - { label: 'Mature', value: 'mature' }, - { label: 'Monstre', value: 'monstre' }, - { label: 'Murim', value: 'murim' }, - { label: 'mystère', value: 'mystere' }, - { label: 'Mystery', value: 'mystery' }, - { label: 'Necromancien', value: 'necromancien' }, - { label: 'necromencer', value: 'necromencer' }, - { label: 'Novel', value: 'novel' }, - { label: 'One Shot', value: 'one-shot' }, - { label: 'Over Power MC', value: 'over-power-mc' }, - { label: 'Partenaire', value: 'partenaire' }, - { label: 'Player', value: 'player' }, - { label: 'Player Manhwa', value: 'player-manhwa' }, - { label: 'Portail', value: 'portail' }, - { label: 'Psychological', value: 'psychological' }, - { label: 'Psychologique', value: 'psychologique' }, - { label: 'regresseur', value: 'regresseur' }, - { label: 'régression', value: 'regression' }, - { label: 'Réincarnation', value: 'reincarnation' }, - { label: 'Returner', value: 'returner' }, - { label: 'Romance', value: 'romance' }, - { label: 'School Life', value: 'school-life' }, - { label: 'Sci-fi', value: 'sci-fi' }, - { label: 'Seinen', value: 'seinen' }, - { label: 'Shôjo', value: 'shojo' }, - { label: 'Shônen', value: 'shonen' }, - { label: 'Shotacon', value: 'shotacon' }, - { label: 'Shoujo', value: 'shoujo' }, - { label: 'Shounen', value: 'shounen' }, - { label: 'Slice of Life', value: 'slice-of-life' }, - { label: 'Slide of Life', value: 'slide-of-life' }, - { label: 'Smut', value: 'smut' }, - { label: 'Sports', value: 'sports' }, - { label: 'Supernatural', value: 'supernatural' }, - { label: 'Surnaturel', value: 'surnaturel' }, - { label: 'Système', value: 'systeme' }, - { label: 'Tragédie', value: 'tragedie' }, - { label: 'Tragedy', value: 'tragedy' }, - { label: 'Webtoons', value: 'webtoons' }, - ], - }, - } satisfies Filters; -} - -export default new PhenixScansTradPlugin(); diff --git a/plugins/french/rezerowebnovelfr.ts b/plugins/french/rezerowebnovelfr.ts index 8d56e2237..a734f8a6e 100644 --- a/plugins/french/rezerowebnovelfr.ts +++ b/plugins/french/rezerowebnovelfr.ts @@ -3,6 +3,33 @@ import { fetchApi } from '@libs/fetch'; import { Plugin } from '@/types/plugin'; import { NovelStatus } from '@libs/novelStatus'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + const NOVEL_METADATA: Record = { '/histoire-principale/': { name: 'Re:Zero - Histoire Principale', @@ -626,15 +653,14 @@ class ReZeroWebNovelFrPlugin implements Plugin.PluginBase { name = 'Re:Zero Web Novel FR'; icon = 'src/fr/rezerowebnovelfr/icon.png'; site = 'https://rezerowebnovelfr.wordpress.com'; - version = '1.0.1'; + version = '1.0.3'; async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); - const body = await r.text(); - const $ = load(body); - return $; + return load( + await fetchCheckedHtml(url, { + headers: { 'Accept-Encoding': 'deflate' }, + }), + ); } async popularNovels(pageNo: number): Promise { @@ -656,7 +682,7 @@ class ReZeroWebNovelFrPlugin implements Plugin.PluginBase { const $ = await this.getCheerio(this.site + novelPath); novel.name = $('h1.entry-title').text().trim(); novel.author = 'Tappei Nagatsuki'; - novel.status = NovelStatus.Ongoing; + novel.status = NovelStatus.Unknown; const meta = NOVEL_METADATA[novelPath]; if (meta) { @@ -666,6 +692,7 @@ class ReZeroWebNovelFrPlugin implements Plugin.PluginBase { } const chapters: Plugin.ChapterItem[] = []; + const chapterPaths = new Set(); const tryAddChapter = ( href: string | undefined, @@ -677,7 +704,8 @@ class ReZeroWebNovelFrPlugin implements Plugin.PluginBase { const dateMatch = cleanHref.match(/\/(\d{4})\/(\d{2})\/(\d{2})\//); if (dateMatch && name) { const path = cleanHref.replace(this.site, ''); - if (!chapters.some(c => c.path === path)) { + if (!chapterPaths.has(path)) { + chapterPaths.add(path); const releaseDate = `${dateMatch[1]}-${dateMatch[2]}-${dateMatch[3]}`; let cleanName = name; if (novelPath === '/if-stories/') { @@ -806,6 +834,7 @@ class ReZeroWebNovelFrPlugin implements Plugin.PluginBase { $( 'div.entry-content .sharedaddy, div.entry-content .wpcnt, div.entry-content #jp-post-flair, div.entry-content div[id^="atatags-"]', ).remove(); + $('div.entry-content').find('script, style, ins, iframe, .ads').remove(); const title = $('h1.entry-title').html() || ''; const chapter = $('div.entry-content').html() || ''; diff --git a/plugins/french/tradindex.ts b/plugins/french/tradindex.ts new file mode 100644 index 000000000..632f7ad58 --- /dev/null +++ b/plugins/french/tradindex.ts @@ -0,0 +1,285 @@ +import { load } from 'cheerio'; +import { defaultCover } from '@libs/defaultCover'; +import { fetchApi } from '@libs/fetch'; +import { NovelStatus } from '@libs/novelStatus'; +import { Plugin } from '@/types/plugin'; + +const catalogueTypes = ['Web Novel', 'Light Novel', 'Manhwa']; +const chapterPathPattern = + /^(?:https?:\/\/trad-index\.com)?\/oeuvre\/([^/?#]+)\/chapitre\/(\d+(?:[.,]\d+)?)/; + +class TradIndexPlugin implements Plugin.PluginBase { + id = 'tradindex'; + name = 'Trad-Index'; + icon = 'src/fr/tradindex/icon.png'; + site = 'https://trad-index.com/'; + version = '1.0.8'; + + // The app injects its own device User-Agent; send a desktop one so the + // server-rendered catalogue/chapter pages stay identical on mobile. + private readonly browserHeaders = { + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36', + }; + + resolveUrl(path: string, isNovel = false): string { + const url = new URL(path, this.site); + if (url.origin !== new URL(this.site).origin) + throw new Error('Cannot resolve a foreign origin'); + const cleanPath = url.pathname + .replace(/^\/+|\/+$/g, '') + .replace(/^oeuvre\//, ''); + if (isNovel) return new URL(`/oeuvre/${cleanPath}`, this.site).href; + + const [slug, chapterNumber] = cleanPath + .replace(/\/chapitre\//, '/') + .split('/'); + return new URL(`/oeuvre/${slug}/chapitre/${chapterNumber}`, this.site).href; + } + + private retryDelay(response: Response, attempt: number): number { + const retryAfter = response.headers.get('retry-after'); + if (retryAfter) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000; + const date = Date.parse(retryAfter); + if (!Number.isNaN(date)) return Math.max(date - Date.now(), 0); + } + return 100 * (attempt + 1); + } + + private async fetchHtml(path: string, retry = false): Promise { + const attempts = retry ? 4 : 1; + const url = new URL(path, this.site).href; + for (let attempt = 0; attempt < attempts; attempt += 1) { + const response = await fetchApi(url, { headers: this.browserHeaders }); + if (response.ok) return response.text(); + if ( + !retry || + (response.status !== 429 && response.status < 500) || + attempt === attempts - 1 + ) + throw new Error(`Failed to load ${path}`); + await new Promise(resolve => + setTimeout(resolve, this.retryDelay(response, attempt)), + ); + } + throw new Error(`Failed to load ${path}`); + } + + private async catalogueSections(paths: string[]): Promise { + const results = await Promise.allSettled( + paths.map(path => this.fetchHtml(path)), + ); + const pages = results.flatMap(result => + result.status === 'fulfilled' ? [result.value] : [], + ); + if (!pages.length) throw new Error('Failed to load catalogue'); + return pages; + } + + private parseCards(html: string): Plugin.NovelItem[] { + const $ = load(html); + const novels = new Map(); + $('a[href]').each((_, element) => { + const href = $(element).attr('href') || ''; + const match = href.match( + /^(?:https?:\/\/trad-index\.com)?\/oeuvre\/([^/?#]+)\/?$/, + ); + if (!match) return; + + const name = $(element) + .find('[class*="line-clamp"]') + .first() + .text() + .trim(); + if (!name) return; + const cover = $(element).find('img').first().attr('src'); + novels.set(match[1], { + name, + path: match[1], + cover: cover ? new URL(cover, this.site).href : defaultCover, + }); + }); + return Array.from(novels.values()); + } + + private cataloguePath(type: string, pageNo: number, searchTerm?: string) { + const query = searchTerm ? `&q=${encodeURIComponent(searchTerm)}` : ''; + return `/catalogue?type=${encodeURIComponent(type)}${query}&page=${pageNo}`; + } + + private chapterItems(html: string, slug: string): Plugin.ChapterItem[] { + const $ = load(html); + const chapters = new Map(); + $('a[href]').each((_, element) => { + const href = $(element).attr('href') || ''; + const match = href.match(chapterPathPattern); + if (!match || match[1] !== slug) return; + + const number = Number(match[2].replace(',', '.')); + if (!Number.isFinite(number)) return; + const path = `${slug}/${match[2]}`; + chapters.set(path, { + name: $(element).text().trim() || `Chapitre ${match[2]}`, + path, + chapterNumber: number, + }); + }); + return Array.from(chapters.values()); + } + + private async fetchChapterPages( + html: string, + slug: string, + ): Promise { + const $ = load(html); + let lastPage = 1; + $('a[href*="onglet=chapitres"]').each((_, element) => { + const href = $(element).attr('href'); + if (!href) return; + const page = Number( + new URL(href, this.resolveUrl(slug, true)).searchParams.get('page'), + ); + if (Number.isInteger(page)) lastPage = Math.max(lastPage, page); + }); + + const pages = await Promise.all( + Array.from({ length: lastPage - 1 }, (_, index) => + this.fetchHtml( + `/oeuvre/${slug}?onglet=chapitres&tri=desc&page=${index + 2}`, + true, + ), + ), + ); + const chapters = new Map(); + for (const pageHtml of [html, ...pages]) { + for (const chapter of this.chapterItems(pageHtml, slug)) { + chapters.set(chapter.path, chapter); + } + } + return Array.from(chapters.values()).sort( + (left, right) => (left.chapterNumber || 0) - (right.chapterNumber || 0), + ); + } + + async popularNovels(pageNo: number): Promise { + const sitePage = Math.max(1, pageNo); + const pages = await this.catalogueSections( + catalogueTypes.map(type => this.cataloguePath(type, sitePage)), + ); + const novels = Array.from( + new Map( + pages + .flat() + .flatMap(html => this.parseCards(html)) + .map(novel => [novel.path, novel]), + ).values(), + ); + if (sitePage === 1 && !novels.length) + throw new Error('Trad-Index catalogue returned no work cards'); + return novels; + } + + async searchNovels( + searchTerm: string, + pageNo: number, + ): Promise { + const pages = await this.catalogueSections( + catalogueTypes.map(type => this.cataloguePath(type, pageNo, searchTerm)), + ); + return Array.from( + new Map( + pages + .flatMap(html => this.parseCards(html)) + .map(novel => [novel.path, novel]), + ).values(), + ); + } + + async parseNovel(novelPath: string): Promise { + const novelUrl = this.resolveUrl(novelPath, true); + const slug = new URL(novelUrl).pathname.replace(/^\/oeuvre\//, ''); + const html = await this.fetchHtml(novelUrl, true); + const $ = load(html); + const details = $('body').text().replace(/\s+/g, ' '); + const formatStatus = details.match( + /(Web Novel|Light Novel|Manhwa)\s*·\s*([^\n]+)/i, + ); + const getDetail = (label: string) => { + let value: string | undefined; + $('*').each((_, element) => { + const text = $(element).text().trim(); + const match = text.match(new RegExp(`^${label}\\s*:\\s*(.+)$`, 'i')); + if (match) { + value = match[1].trim(); + return false; + } + }); + return value; + }; + const synopsisHeading = $('h1,h2,h3') + .filter( + (_, element) => $(element).text().trim().toLowerCase() === 'synopsis', + ) + .first(); + + return { + path: slug, + name: $('h1').first().text().trim(), + cover: $('img[alt^="Couverture de"]').first().attr('src') + ? new URL( + $('img[alt^="Couverture de"]').first().attr('src')!, + this.site, + ).href + : defaultCover, + summary: synopsisHeading.nextAll('p').first().text().trim() || undefined, + author: getDetail('Auteur'), + artist: getDetail('Traducteur'), + genres: getDetail('Genres'), + status: /terminé/i.test(formatStatus?.[2] || '') + ? NovelStatus.Completed + : /en cours/i.test(formatStatus?.[2] || '') + ? NovelStatus.Ongoing + : NovelStatus.Unknown, + chapters: await this.fetchChapterPages(html, slug), + }; + } + + async parseChapter(chapterPath: string): Promise { + const html = await this.fetchHtml( + this.resolveUrl(chapterPath).replace(this.site.slice(0, -1), ''), + ); + const $ = load(html); + const main = $('main').first(); + const stopPattern = + /traduit par|traducteur|navigation|partager|signaler|commentaires?/i; + let stopped = false; + const prose: string[] = []; + main + .find( + 'h1, h2, h3, h4, h5, h6, p, nav, form, [class*="comment"], [class*="share"], [class*="report"], [class*="translator"]', + ) + .each((_, element) => { + const part = $(element); + if (stopped) return false; + if (stopPattern.test(part.text())) { + stopped = true; + return false; + } + if ( + element.tagName === 'p' && + (part.hasClass('narration') || part.hasClass('dialogue')) + ) + prose.push($.html(element)); + }); + + const content = prose.join(''); + if (load(content).text().trim().length < 200) + throw new Error('No readable chapter content found'); + return content; + } +} + +export default new TradIndexPlugin(); diff --git a/plugins/french/warriorlegendtrad.ts b/plugins/french/warriorlegendtrad.ts index b411b424d..f9e579fb0 100644 --- a/plugins/french/warriorlegendtrad.ts +++ b/plugins/french/warriorlegendtrad.ts @@ -5,12 +5,39 @@ import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; import dayjs from 'dayjs'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class WarriorLegendTradPlugin implements Plugin.PluginBase { id = 'warriorlegendtrad'; name = 'Warrior Legend Trad'; icon = 'src/fr/warriorlegendtrad/icon.png'; site = 'https://warriorlegendtrad.wordpress.com'; - version = '1.0.1'; + version = '1.0.4'; regexAuthors = [/Auteur\u00A0:([^\n]*)/]; @@ -19,10 +46,7 @@ class WarriorLegendTradPlugin implements Plugin.PluginBase { regexSummary = [/Synopsis\u00A0:([\s\S]*)index chapitre :/i]; async getCheerio(url: string): Promise { - const r = await fetchApi(url); - const body = await r.text(); - const $ = load(body); - return $; + return load(await fetchCheckedHtml(url)); } async popularNovels(pageNo: number): Promise { @@ -84,7 +108,7 @@ class WarriorLegendTradPlugin implements Plugin.PluginBase { const chapterUrl = $(elem).attr('href'); const releaseDate = dayjs( chapterUrl?.substring(this.site.length + 1, this.site.length + 11), - ).format('DD MMMM YYYY'); + ).format('YYYY-MM-DD'); if (chapterUrl && chapterUrl.includes(this.site) && chapterName) { chapters.push({ name: chapterName, @@ -137,11 +161,12 @@ class WarriorLegendTradPlugin implements Plugin.PluginBase { return NovelStatus.Cancelled; } - return NovelStatus.Ongoing; + return NovelStatus.Unknown; } async parseChapter(chapterPath: string): Promise { const $ = await this.getCheerio(this.site + chapterPath); + $('.entry-content').find('script, style, ins, iframe, .ads').remove(); let contenuHtml = ''; $('.entry-content') .contents() @@ -163,9 +188,11 @@ class WarriorLegendTradPlugin implements Plugin.PluginBase { ): Promise { if (pageNo !== 1) return []; - const popularNovels = this.popularNovels(1); + const popularNovels = ( + await Promise.all([this.popularNovels(1), this.popularNovels(2)]) + ).flat(); - const novels = (await popularNovels).filter(novel => + const novels = popularNovels.filter(novel => novel.name .toLowerCase() .normalize('NFD') diff --git a/plugins/french/wuxialnscantrad.ts b/plugins/french/wuxialnscantrad.ts index 5789ec2b8..29fc756b3 100644 --- a/plugins/french/wuxialnscantrad.ts +++ b/plugins/french/wuxialnscantrad.ts @@ -5,20 +5,70 @@ import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; import dayjs from 'dayjs'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class WuxialnscantradPlugin implements Plugin.PluginBase { id = 'wuxialnscantrad'; name = 'WuxiaLnScantrad'; icon = 'src/fr/wuxialnscantrad/icon.png'; site = 'https://wuxialnscantrad.wordpress.com'; - version = '1.0.0'; + version = '1.0.4'; + + private async findMovedChapter(chapterPath: string): Promise { + const slug = chapterPath.split('/').filter(Boolean).pop() || ''; + const match = slug.match(/^(.*?)-chapitre-(\d+)/i); + if (!match) return null; + const series = match[1].split('-').filter(Boolean).slice(0, 3).join(' '); + const query = `${series} chapitre ${match[2]}`; + const response = await fetchApi( + `https://public-api.wordpress.com/wp/v2/sites/wuxialnscantrad.wordpress.com/search?search=${encodeURIComponent(query)}&type=post&subtype=post&per_page=20`, + ); + if (!response.ok) return null; + const results = (await response.json()) as { + title?: string; + url?: string; + }[]; + const replacement = results.find(result => + new RegExp(`chapitre\\D*${match[2]}(?:\\D|$)`, 'i').test( + result.title || '', + ), + )?.url; + if (!replacement) return null; + const url = new URL(replacement, this.site); + return url.origin === new URL(this.site).origin ? url.toString() : null; + } async getCheerio(url: string): Promise { - const r = await fetchApi(url, { - headers: { 'Accept-Encoding': 'deflate' }, - }); - const body = await r.text(); - const $ = load(body); - return $; + return load( + await fetchCheckedHtml(url, { + headers: { 'Accept-Encoding': 'deflate' }, + }), + ); } async popularNovels(pageNo: number): Promise { @@ -41,6 +91,15 @@ class WuxialnscantradPlugin implements Plugin.PluginBase { novels.push(novel); } }); + await Promise.all( + novels.map(async item => { + const detail = await this.getCheerio(this.site + item.path); + item.cover = + detail('.entry-content p strong img').first().attr('src') || + detail('.entry-content p img').first().attr('src') || + defaultCover; + }), + ); return novels; } @@ -66,16 +125,17 @@ class WuxialnscantradPlugin implements Plugin.PluginBase { const pathChapter = $('.entry-content ul').first().children('li'); const chapters: Plugin.ChapterItem[] = []; + const chapterPaths = new Set(); pathChapter.each((i, elem) => { const chapterName = $(elem).text().trim(); const chapterUrl = $(elem).find('a').attr('href'); if (chapterUrl && chapterUrl.includes(this.site) && chapterName) { const pathchapter = chapterUrl.replace(this.site, ''); - // we do not take the paths already present - if (!chapters.some(chap => chap.path === pathchapter)) { + if (!chapterPaths.has(pathchapter)) { + chapterPaths.add(pathchapter); const releaseDate = dayjs( chapterUrl?.substring(this.site.length + 1, this.site.length + 11), - ).format('DD MMMM YYYY'); + ).format('YYYY-MM-DD'); chapters.push({ name: chapterName, path: pathchapter, @@ -149,12 +209,21 @@ class WuxialnscantradPlugin implements Plugin.PluginBase { case 'Terminé': return NovelStatus.Completed; default: - return NovelStatus.Ongoing; + return NovelStatus.Unknown; } } async parseChapter(chapterPath: string): Promise { - const $ = await this.getCheerio(this.site + chapterPath); + const options = { headers: { 'Accept-Encoding': 'deflate' } }; + let body: string; + try { + body = await fetchCheckedHtml(this.site + chapterPath, options); + } catch (error) { + const replacement = await this.findMovedChapter(chapterPath); + if (!replacement) throw error; + body = await fetchCheckedHtml(replacement, options); + } + const $ = load(body); let contenuHtml = ''; $('.entry-content') @@ -172,6 +241,12 @@ class WuxialnscantradPlugin implements Plugin.PluginBase { contenuHtml += $.html(this); } }); + const parsedContent = load(contenuHtml); + if ( + parsedContent.text().replace(/\s+/g, ' ').trim().length < 200 && + !parsedContent('img').length + ) + throw new Error('No readable chapter content found'); return contenuHtml; } diff --git a/plugins/french/xiaowaz.ts b/plugins/french/xiaowaz.ts index 5d3a09e50..5563d68c1 100644 --- a/plugins/french/xiaowaz.ts +++ b/plugins/french/xiaowaz.ts @@ -5,12 +5,39 @@ import { Plugin } from '@/types/plugin'; import { defaultCover } from '@libs/defaultCover'; import { NovelStatus } from '@libs/novelStatus'; +const challengeTitles = new Set([ + 'bot verification', + 'you are being redirected...', + 'un instant...', + 'just a moment...', + 'redirecting...', +]); + +async function fetchCheckedHtml( + url: string, + init?: Parameters[1], +): Promise { + const response = await fetchApi(url, init); + if (!response.ok) + throw new Error(`HTTP ${response.status} while loading ${url}`); + const html = await response.text(); + const title = html + .match(/]*>(.*?)<\/title>/is)?.[1] + ?.replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + if (title && challengeTitles.has(title)) + throw new Error(`Bot challenge while loading ${url}`); + return html; +} + class XiaowazPlugin implements Plugin.PluginBase { id = 'xiaowaz'; name = 'Xiaowaz'; icon = 'src/fr/xiaowaz/icon.png'; site = 'https://xiaowaz.fr'; - version = '1.0.2'; + version = '1.0.5'; static novels: Plugin.NovelItem[] | undefined; async getCheerio(url: string): Promise { @@ -18,10 +45,7 @@ class XiaowazPlugin implements Plugin.PluginBase { let returnError: unknown; while (retries > 0) { try { - const r = await fetchApi(url); - const body = await r.text(); - const $ = load(body); - return $; + return load(await fetchCheckedHtml(url)); } catch (error) { console.error(error); returnError = error; @@ -171,13 +195,22 @@ class XiaowazPlugin implements Plugin.PluginBase { } const chapters: Plugin.ChapterItem[] = []; + const chapterPaths = new Set(); pathChapter.each((i, elem) => { const chapterName = $(elem).text().trim(); const chapterUrl = $(elem).attr('href'); - if (chapterUrl && chapterUrl.includes(this.site) && chapterName) { + if ( + chapterUrl && + chapterUrl.includes(this.site) && + !/\.pdf(?:$|[?#])/i.test(chapterUrl) && + chapterName + ) { + const path = chapterUrl.replace(this.site, ''); + if (chapterPaths.has(path)) return; + chapterPaths.add(path); chapters.push({ name: chapterName, - path: chapterUrl.replace(this.site, ''), + path, }); } }); @@ -229,15 +262,20 @@ class XiaowazPlugin implements Plugin.PluginBase { async parseChapter(chapterPath: string): Promise { const $ = await this.getCheerio(this.site + chapterPath); - const startTag = $('.wp-post-navigation'); - const endTag = $('.abh_box.abh_box_down.abh_box_business'); + const startTag = $('.entry-content .wp-post-navigation').first(); const elementsBetweenTags: string[] = []; let footnotesElement: string | null = null; - if (startTag.length > 0 && endTag.length > 0) { + if (startTag.length > 0) { let currentElement = startTag.next(); - while (currentElement.length > 0 && !currentElement.is(endTag)) { + while (currentElement.length > 0) { + if ( + currentElement.hasClass('wp-post-navigation') || + currentElement.is('.abh_box.abh_box_down.abh_box_business') + ) { + break; + } if ( currentElement.find('p > a[href="https://ko-fi.com/wazouille"]') .length > 0 diff --git a/public/static/src/fr/jgarden/icon.png b/public/static/src/fr/jgarden/icon.png new file mode 100644 index 000000000..0fe9c5f14 Binary files /dev/null and b/public/static/src/fr/jgarden/icon.png differ diff --git a/public/static/src/fr/lightnovelvf/icon.png b/public/static/src/fr/lightnovelvf/icon.png new file mode 100644 index 000000000..275e81673 Binary files /dev/null and b/public/static/src/fr/lightnovelvf/icon.png differ diff --git a/public/static/src/fr/phenixscans/icon.png b/public/static/src/fr/phenixscans/icon.png deleted file mode 100644 index e4a836c0b..000000000 Binary files a/public/static/src/fr/phenixscans/icon.png and /dev/null differ diff --git a/public/static/src/fr/tradindex/icon.png b/public/static/src/fr/tradindex/icon.png new file mode 100644 index 000000000..36567fc68 Binary files /dev/null and b/public/static/src/fr/tradindex/icon.png differ