Initial commit: ML scraper service (Playwright + stealth + Redis cache)

.gitignore

@@ -0,0 +1,2 @@
+.env
+node_modules/

Dockerfile

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/playwright:v1.45.0-jammy
+WORKDIR /app
+COPY package.json ./
+RUN npm install
+RUN npx playwright install chromium --with-deps
+COPY server.js ./
+EXPOSE 3000
+CMD ["node", "server.js"]

docker-compose.yml

@@ -0,0 +1,41 @@
+version: "3.8"
+
+services:
+  ml-scraper:
+    build: .
+    restart: unless-stopped
+    depends_on:
+      - redis-ml
+    environment:
+      REDIS_URL: "redis://redis-ml:6379/0"
+      API_KEY: "${ML_API_KEY}"
+      CACHE_TTL_SECONDS: "7200"
+      PORT: "3000"
+    networks:
+      - internal
+      - coolify
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.ml-scraper.rule=Host(`mlsearch.bizpilot.com.br`)"
+      - "traefik.http.routers.ml-scraper.entrypoints=websecure"
+      - "traefik.http.routers.ml-scraper.tls.certresolver=letsencrypt"
+      - "traefik.http.services.ml-scraper.loadbalancer.server.port=3000"
+      - "traefik.docker.network=coolify"
+
+  redis-ml:
+    image: redis:7-alpine
+    restart: unless-stopped
+    volumes:
+      - redis_ml_data:/data
+    networks:
+      - internal
+    command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru
+
+volumes:
+  redis_ml_data:
+
+networks:
+  internal:
+    driver: bridge
+  coolify:
+    external: true

package.json

@@ -0,0 +1,12 @@
+{
+  "name": "ml-scraper",
+  "version": "1.0.0",
+  "main": "server.js",
+  "dependencies": {
+    "express": "^4.18.2",
+    "ioredis": "^5.3.2",
+    "playwright-extra": "^0.0.7",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "playwright": "^1.45.0"
+  }
+}

server.js

@@ -0,0 +1,217 @@
+const express = require('express');
+const { chromium } = require('playwright-extra');
+const stealth = require('puppeteer-extra-plugin-stealth');
+const Redis = require('ioredis');
+
+chromium.use(stealth());
+
+const app = express();
+app.use(express.json());
+
+const redis = new Redis(process.env.REDIS_URL || 'redis://redis-ml:6379/0');
+const API_KEY = process.env.API_KEY;
+const CACHE_TTL = parseInt(process.env.CACHE_TTL_SECONDS || '7200');
+const COOKIE_TTL = 8 * 60 * 60; // 8 hours
+const COOKIE_KEY = 'ml:session:cookies';
+
+let refreshing = false;
+
+// Auth
+app.use((req, res, next) => {
+  if (req.path === '/health') return next();
+  if (API_KEY && req.headers['x-api-key'] !== API_KEY) {
+    return res.status(401).json({ error: 'Unauthorized' });
+  }
+  next();
+});
+
+app.get('/health', (req, res) => res.json({ status: 'ok' }));
+
+// Cookie refresh via Playwright (runs rarely — ~once per 6h)
+async function refreshCookies() {
+  if (refreshing) return;
+  refreshing = true;
+  console.log('[cookies] starting refresh...');
+  let browser;
+  try {
+    browser = await chromium.launch({
+      headless: true,
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
+    });
+
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+      viewport: { width: 1280, height: 800 },
+      locale: 'pt-BR',
+      timezoneId: 'America/Sao_Paulo',
+    });
+
+    const page = await context.newPage();
+    await page.goto('https://lista.mercadolivre.com.br/air-fryer', {
+      waitUntil: 'domcontentloaded',
+      timeout: 30000,
+    });
+    await page.waitForTimeout(3000);
+
+    const cookies = await context.cookies();
+    const cookieHeader = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+    await redis.setex(COOKIE_KEY, COOKIE_TTL, cookieHeader);
+    console.log(`[cookies] refreshed — ${cookies.length} cookies saved`);
+    return cookieHeader;
+  } catch (e) {
+    console.error('[cookies] refresh failed:', e.message);
+    return null;
+  } finally {
+    if (browser) await browser.close().catch(() => {});
+    refreshing = false;
+  }
+}
+
+async function getCookies() {
+  const cached = await redis.get(COOKIE_KEY).catch(() => null);
+  if (cached) return cached;
+  return refreshCookies();
+}
+
+// Parse products from ML's __PRELOADED_STATE__ JSON (more stable than CSS selectors)
+function parsePreloadedState(html) {
+  const match = html.match(/<script[^>]*>\s*window\.__PRELOADED_STATE__\s*=\s*(\{[\s\S]*?\});\s*<\/script>/);
+  if (!match) return null;
+  try {
+    const state = JSON.parse(match[1]);
+    const results =
+      state?.initialState?.results ||
+      state?.results ||
+      state?.listingMain?.results ||
+      null;
+    if (!Array.isArray(results)) return null;
+
+    return results.map(r => ({
+      title: r.title || r.name || '',
+      price: r.price ? `R$ ${String(r.price).replace('.', ',')}` : null,
+      url: (r.permalink || r.url || '').split('?')[0],
+      imageUrl: r.thumbnail || r.pictures?.[0]?.url || null,
+    })).filter(p => p.title && p.url);
+  } catch {
+    return null;
+  }
+}
+
+// Fallback: parse poly-cards from raw HTML (no browser, just regex/string)
+function parseHtmlFallback(html) {
+  const products = [];
+  // Extract JSON-LD product data if present
+  const jsonldMatches = [...html.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
+  for (const m of jsonldMatches) {
+    try {
+      const d = JSON.parse(m[1]);
+      if (d['@type'] === 'Product' && d.name && d.offers?.url) {
+        products.push({
+          title: d.name,
+          price: d.offers?.price ? `R$ ${d.offers.price}` : null,
+          url: d.offers.url.split('?')[0],
+          imageUrl: d.image || null,
+        });
+      }
+    } catch {}
+  }
+  return products;
+}
+
+async function searchML(term, limit) {
+  const cookies = await getCookies();
+  if (!cookies) throw new Error('Não foi possível obter cookies do ML');
+
+  const slug = encodeURIComponent(term.replace(/\s+/g, '-'));
+  const url = `https://lista.mercadolivre.com.br/${slug}`;
+
+  const res = await fetch(url, {
+    headers: {
+      'Cookie': cookies,
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+      'Accept-Language': 'pt-BR,pt;q=0.9',
+      'Accept-Encoding': 'gzip, deflate, br',
+      'Referer': 'https://www.mercadolivre.com.br/',
+      'Sec-Fetch-Dest': 'document',
+      'Sec-Fetch-Mode': 'navigate',
+      'Sec-Fetch-Site': 'same-site',
+    },
+    signal: AbortSignal.timeout(15000),
+  });
+
+  if (res.status === 403 || res.status === 429) {
+    // Cookies expired — schedule refresh and throw so caller can retry
+    refreshCookies().catch(() => {});
+    throw new Error(`ML retornou ${res.status} — cookies expirados, atualizando...`);
+  }
+
+  const html = await res.text();
+
+  let products = parsePreloadedState(html);
+  if (!products || products.length === 0) {
+    products = parseHtmlFallback(html);
+  }
+
+  if (!products || products.length === 0) {
+    throw new Error('Nenhum produto encontrado no HTML retornado');
+  }
+
+  return products.slice(0, limit);
+}
+
+function injectTag(products, tag) {
+  if (!tag) return products;
+  return products.map(p => ({ ...p, url: `${p.url}?tag=${encodeURIComponent(tag)}` }));
+}
+
+// POST /search
+app.post('/search', async (req, res) => {
+  const { q, limit = 5, tag } = req.body;
+  if (!q?.trim()) return res.status(400).json({ error: 'Parâmetro q obrigatório' });
+
+  const term = q.trim();
+  const cacheKey = `ml:search:${term.toLowerCase()}:${limit}`;
+
+  try {
+    const cached = await redis.get(cacheKey);
+    if (cached) {
+      return res.json({ products: injectTag(JSON.parse(cached), tag), cached: true });
+    }
+  } catch (_) {}
+
+  try {
+    let products = await searchML(term, limit);
+
+    // If 403, retry once after a short wait (cookies should be refreshing)
+    if (!products) {
+      await new Promise(r => setTimeout(r, 5000));
+      products = await searchML(term, limit);
+    }
+
+    await redis.setex(cacheKey, CACHE_TTL, JSON.stringify(products)).catch(() => {});
+    return res.json({ products: injectTag(products, tag), cached: false });
+  } catch (e) {
+    return res.status(500).json({ error: e.message, products: [] });
+  }
+});
+
+// POST /refresh-cookies — manual trigger
+app.post('/refresh-cookies', async (req, res) => {
+  const result = await refreshCookies();
+  if (result) return res.json({ ok: true, message: 'Cookies atualizados' });
+  return res.status(500).json({ ok: false, message: 'Falha ao atualizar cookies' });
+});
+
+// Auto-refresh cookies every 6 hours
+setInterval(() => {
+  refreshCookies().catch(e => console.error('[cron] cookie refresh error:', e.message));
+}, 6 * 60 * 60 * 1000);
+
+// Warm up cookies on startup
+setTimeout(() => {
+  getCookies().catch(e => console.error('[startup] cookie warm-up error:', e.message));
+}, 3000);
+
+const PORT = process.env.PORT || 3000;
+app.listen(PORT, () => console.log(`ML Scraper rodando na porta ${PORT}`));