chore: add CI, test suite and English codebase

.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install dependencies
+        run: pip install -r requirements-dev.txt
+      - name: Lint
+        run: ruff check . && ruff format --check .
+      - name: Run tests
+        run: pytest -v

README.md

@@ -4,6 +4,7 @@
 
 **A complete web scraping kit with BeautifulSoup, Selenium, and Playwright**
 
+[![CI](https://github.com/LacerdaTraderCode/web-scraper-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/LacerdaTraderCode/web-scraper-toolkit/actions/workflows/ci.yml)
 [![Python](https://img.shields.io/badge/Python-3.11%2B-blue?logo=python&logoColor=white)](https://www.python.org/)
 [![Selenium](https://img.shields.io/badge/Selenium-43B02A?logo=selenium&logoColor=white)](https://www.selenium.dev/)
 [![Playwright](https://img.shields.io/badge/Playwright-2EAD33?logo=playwright&logoColor=white)](https://playwright.dev/)
@@ -57,7 +58,10 @@ web-scraper-toolkit/
 │   ├── quotes_bs4.py         # Scraping quotes.toscrape.com
 │   ├── books_selenium.py     # Scraping books.toscrape.com
 │   └── news_playwright.py    # News scraping (async)
+├── tests/
+├── .github/workflows/ci.yml
 ├── requirements.txt
+├── requirements-dev.txt
 └── README.md
 ```
 
@@ -85,19 +89,19 @@ playwright install chromium
 
 ### BeautifulSoup — static HTML
 ```bash
-python examples/quotes_bs4.py
+python -m examples.quotes_bs4
 ```
 Extracts quotes from `quotes.toscrape.com`, saves to CSV and Parquet.
 
 ### Selenium — JavaScript-heavy sites
 ```bash
-python examples/books_selenium.py
+python -m examples.books_selenium
 ```
 Navigates the `books.toscrape.com` catalog simulating a real user.
 
 ### Playwright — asynchronous scraping
 ```bash
-python examples/news_playwright.py
+python -m examples.news_playwright
 ```
 Collects headlines asynchronously (faster than Selenium).
 
@@ -113,6 +117,18 @@ Collects headlines asynchronously (faster than Selenium).
 
 ---
 
+## 🧪 Development
+
+```bash
+pip install -r requirements-dev.txt
+ruff check . && ruff format --check .
+pytest -v
+```
+
+Browser drivers and HTTP sessions are mocked, so the test suite needs no browser or network access. The same checks run on every push and pull request via GitHub Actions.
+
+---
+
 ## ⚖️ Ethical Use
 
 This toolkit is for educational purposes. When scraping:

examples/books_selenium.py

@@ -1,18 +1,11 @@
-"""
-Exemplo: scraping de livros usando Selenium.
-Executar: python examples/books_selenium.py
-"""
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
+import logging
 
 from scrapers.selenium_scraper import SeleniumScraper
 from utils.exporters import export_all
 
 
 def main():
-    print("🚀 Iniciando scraping com Selenium...\n")
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 
     scraper = SeleniumScraper(headless=True)
     try:
@@ -21,8 +14,6 @@ def main():
     finally:
         scraper.close()
 
-    print("\n✅ Concluído!")
-
 
 if __name__ == "__main__":
     main()

examples/news_playwright.py

@@ -1,26 +1,17 @@
-"""
-Exemplo: scraping assíncrono com Playwright.
-Executar: python examples/news_playwright.py
-"""
 import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
+import logging
 
 from scrapers.playwright_scraper import PlaywrightScraper
 from utils.exporters import export_all
 
 
 async def main():
-    print("🚀 Iniciando scraping assíncrono com Playwright...\n")
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 
     scraper = PlaywrightScraper()
     quotes = await scraper.scrape_quotes_async(max_pages=5)
     export_all(quotes, "output/quotes_playwright")
 
-    print("\n✅ Concluído!")
-
 
 if __name__ == "__main__":
     asyncio.run(main())

examples/quotes_bs4.py

@@ -1,19 +1,11 @@
-"""
-Exemplo: scraping de quotes usando BeautifulSoup.
-Executar: python examples/quotes_bs4.py
-"""
-import sys
-from pathlib import Path
-
-# Adiciona raiz do projeto ao path
-sys.path.insert(0, str(Path(__file__).parent.parent))
+import logging
 
 from scrapers.bs4_scraper import BeautifulSoupScraper
 from utils.exporters import export_all
 
 
 def main():
-    print("🚀 Iniciando scraping com BeautifulSoup...\n")
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
 
     scraper = BeautifulSoupScraper(min_interval=1.0)
     try:
@@ -22,8 +14,6 @@ def main():
     finally:
         scraper.close()
 
-    print("\n✅ Concluído!")
-
 
 if __name__ == "__main__":
     main()

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.ruff]
+line-length = 100
+exclude = [".venv"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "RUF"]

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+python_files = test_*.py
+testpaths = tests
+asyncio_mode = auto

requirements-dev.txt

@@ -0,0 +1,4 @@
+-r requirements.txt
+pytest==9.1.1
+pytest-asyncio==1.4.0
+ruff==0.16.9

scrapers/bs4_scraper.py

@@ -1,27 +1,25 @@
-"""
-Scraper usando BeautifulSoup + Requests para HTML estático.
-"""
+import logging
+
 import requests
 from bs4 import BeautifulSoup
 from tenacity import retry, stop_after_attempt, wait_exponential
-from typing import List, Dict
 
 from utils.rate_limiter import RateLimiter, get_random_user_agent
 
+logger = logging.getLogger(__name__)
 
-class BeautifulSoupScraper:
-    """Scraper simples e rápido para páginas estáticas."""
 
+class BeautifulSoupScraper:
     def __init__(self, min_interval: float = 1.0):
         self.rate_limiter = RateLimiter(min_interval)
         self.session = requests.Session()
 
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
     )
     def fetch(self, url: str) -> BeautifulSoup:
-        """Busca uma URL e retorna objeto BeautifulSoup."""
         self.rate_limiter.wait()
 
         headers = {"User-Agent": get_random_user_agent()}
@@ -30,41 +28,35 @@ def fetch(self, url: str) -> BeautifulSoup:
 
         return BeautifulSoup(response.text, "lxml")
 
-    def scrape_quotes(self, base_url: str = "https://quotes.toscrape.com") -> List[Dict]:
-        """
-        Exemplo: extrai quotes do site quotes.toscrape.com (site oficial pra aprender).
-        """
+    @staticmethod
+    def _parse_quote(div) -> dict:
+        tags = [tag.get_text(strip=True) for tag in div.find_all("a", class_="tag")]
+        return {
+            "text": div.find("span", class_="text").get_text(strip=True),
+            "author": div.find("small", class_="author").get_text(strip=True),
+            "tags": ", ".join(tags),
+        }
+
+    def scrape_quotes(self, base_url: str = "https://quotes.toscrape.com") -> list[dict]:
         quotes = []
         page = 1
 
         while True:
-            url = f"{base_url}/page/{page}/"
-            print(f"📄 Scraping página {page}...")
-
-            soup = self.fetch(url)
+            logger.info("Scraping page %d", page)
+            soup = self.fetch(f"{base_url}/page/{page}/")
             quote_divs = soup.find_all("div", class_="quote")
 
             if not quote_divs:
                 break
 
-            for div in quote_divs:
-                quotes.append({
-                    "text": div.find("span", class_="text").get_text(strip=True),
-                    "author": div.find("small", class_="author").get_text(strip=True),
-                    "tags": ", ".join(
-                        tag.get_text(strip=True)
-                        for tag in div.find_all("a", class_="tag")
-                    ),
-                })
-
-            # Verifica se tem próxima página
-            next_btn = soup.find("li", class_="next")
-            if not next_btn:
+            quotes.extend(self._parse_quote(div) for div in quote_divs)
+
+            if not soup.find("li", class_="next"):
                 break
             page += 1
 
-        print(f"✅ Total coletado: {len(quotes)} quotes")
+        logger.info("Collected %d quotes", len(quotes))
         return quotes
 
-    def close(self):
+    def close(self) -> None:
         self.session.close()

scrapers/playwright_scraper.py

@@ -1,65 +1,54 @@
-"""
-Scraper assíncrono usando Playwright - moderno e rápido.
-"""
 import asyncio
-from typing import List, Dict
-from playwright.async_api import async_playwright
+import logging
 
+from playwright.async_api import Error, async_playwright
 
-class PlaywrightScraper:
-    """Scraper moderno, assíncrono e multi-browser."""
+logger = logging.getLogger(__name__)
 
+EXTRACT_QUOTES_SCRIPT = """
+() => {
+    const items = document.querySelectorAll('.quote');
+    return Array.from(items).map(q => ({
+        text: q.querySelector('.text')?.innerText || '',
+        author: q.querySelector('.author')?.innerText || '',
+        tags: Array.from(q.querySelectorAll('.tag')).map(t => t.innerText).join(', ')
+    }));
+}
+"""
+
+
+class PlaywrightScraper:
     async def scrape_quotes_async(
         self, base_url: str = "https://quotes.toscrape.com", max_pages: int = 5
-    ) -> List[Dict]:
-        """
-        Scraping assíncrono - várias páginas em paralelo.
-        Muito mais rápido que Selenium.
-        """
+    ) -> list[dict]:
         quotes = []
 
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
-            context = await browser.new_context()
-
-            # Criar tasks em paralelo
-            tasks = []
-            for page_num in range(1, max_pages + 1):
-                tasks.append(
-                    self._scrape_page(context, f"{base_url}/page/{page_num}/")
-                )
-
-            results = await asyncio.gather(*tasks, return_exceptions=True)
-
-            for result in results:
-                if isinstance(result, list):
-                    quotes.extend(result)
-
-            await browser.close()
-
-        print(f"✅ Total coletado: {len(quotes)} quotes (async)")
+        async with async_playwright() as playwright:
+            browser = await playwright.chromium.launch(headless=True)
+            try:
+                context = await browser.new_context()
+                tasks = [
+                    self._scrape_page(context, f"{base_url}/page/{page_number}/")
+                    for page_number in range(1, max_pages + 1)
+                ]
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+            finally:
+                await browser.close()
+
+        for result in results:
+            if isinstance(result, list):
+                quotes.extend(result)
+
+        logger.info("Collected %d quotes (async)", len(quotes))
         return quotes
 
-    async def _scrape_page(self, context, url: str) -> List[Dict]:
-        """Extrai quotes de uma única página."""
+    async def _scrape_page(self, context, url: str) -> list[dict]:
         page = await context.new_page()
         try:
             await page.goto(url, wait_until="domcontentloaded", timeout=15000)
-
-            quotes = await page.evaluate("""
-                () => {
-                    const items = document.querySelectorAll('.quote');
-                    return Array.from(items).map(q => ({
-                        text: q.querySelector('.text')?.innerText || '',
-                        author: q.querySelector('.author')?.innerText || '',
-                        tags: Array.from(q.querySelectorAll('.tag'))
-                            .map(t => t.innerText).join(', ')
-                    }));
-                }
-            """)
-            return quotes
-        except Exception as e:
-            print(f"⚠️ Erro em {url}: {e}")
+            return await page.evaluate(EXTRACT_QUOTES_SCRIPT)
+        except Error as exc:
+            logger.warning("Failed to scrape %s: %s", url, exc)
             return []
         finally:
             await page.close()