ml-scraper/server.js

const express = require('express');
const { chromium } = require('playwright-extra');
const stealth = require('puppeteer-extra-plugin-stealth');
const Redis = require('ioredis');

chromium.use(stealth());

const app = express();
app.use(express.json());

const redis = new Redis(process.env.REDIS_URL || 'redis://redis-ml:6379/0');
const API_KEY = process.env.API_KEY;
const CACHE_TTL = parseInt(process.env.CACHE_TTL_SECONDS || '7200');
const COOKIE_TTL = 8 * 60 * 60; // 8 hours
const COOKIE_KEY = 'ml:session:cookies';

let refreshing = false;

// Auth
app.use((req, res, next) => {
  if (req.path === '/health') return next();
  if (API_KEY && req.headers['x-api-key'] !== API_KEY) {
    return res.status(401).json({ error: 'Unauthorized' });
  }
  next();
});

app.get('/health', (req, res) => res.json({ status: 'ok' }));

// Cookie refresh via Playwright (runs rarely — ~once per 6h)
async function refreshCookies() {
  if (refreshing) return;
  refreshing = true;
  console.log('[cookies] starting refresh...');
  let browser;
  try {
    browser = await chromium.launch({
      headless: true,
      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
    });

    const context = await browser.newContext({
      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 800 },
      locale: 'pt-BR',
      timezoneId: 'America/Sao_Paulo',
    });

    const page = await context.newPage();
    await page.goto('https://lista.mercadolivre.com.br/air-fryer', {
      waitUntil: 'domcontentloaded',
      timeout: 30000,
    });
    await page.waitForTimeout(3000);

    const cookies = await context.cookies();
    const cookieHeader = cookies.map(c => `${c.name}=${c.value}`).join('; ');
    await redis.setex(COOKIE_KEY, COOKIE_TTL, cookieHeader);
    console.log(`[cookies] refreshed — ${cookies.length} cookies saved`);
    return cookieHeader;
  } catch (e) {
    console.error('[cookies] refresh failed:', e.message);
    return null;
  } finally {
    if (browser) await browser.close().catch(() => {});
    refreshing = false;
  }
}

async function getCookies() {
  const cached = await redis.get(COOKIE_KEY).catch(() => null);
  if (cached) return cached;
  return refreshCookies();
}

// Parse products from ML's __PRELOADED_STATE__ JSON (more stable than CSS selectors)
function parsePreloadedState(html) {
  const match = html.match(/<script[^>]*>\s*window\.__PRELOADED_STATE__\s*=\s*(\{[\s\S]*?\});\s*<\/script>/);
  if (!match) return null;
  try {
    const state = JSON.parse(match[1]);
    const results =
      state?.initialState?.results ||
      state?.results ||
      state?.listingMain?.results ||
      null;
    if (!Array.isArray(results)) return null;

    return results.map(r => ({
      title: r.title || r.name || '',
      price: r.price ? `R$ ${String(r.price).replace('.', ',')}` : null,
      url: (r.permalink || r.url || '').split('?')[0],
      imageUrl: r.thumbnail || r.pictures?.[0]?.url || null,
    })).filter(p => p.title && p.url);
  } catch {
    return null;
  }
}

// Fallback: parse poly-cards from raw HTML (no browser, just regex/string)
function parseHtmlFallback(html) {
  const products = [];
  // Extract JSON-LD product data if present
  const jsonldMatches = [...html.matchAll(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
  for (const m of jsonldMatches) {
    try {
      const d = JSON.parse(m[1]);
      if (d['@type'] === 'Product' && d.name && d.offers?.url) {
        products.push({
          title: d.name,
          price: d.offers?.price ? `R$ ${d.offers.price}` : null,
          url: d.offers.url.split('?')[0],
          imageUrl: d.image || null,
        });
      }
    } catch {}
  }
  return products;
}

async function searchML(term, limit) {
  const cookies = await getCookies();
  if (!cookies) throw new Error('Não foi possível obter cookies do ML');

  const slug = encodeURIComponent(term.replace(/\s+/g, '-'));
  const url = `https://lista.mercadolivre.com.br/${slug}`;

  const res = await fetch(url, {
    headers: {
      'Cookie': cookies,
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
      'Accept-Language': 'pt-BR,pt;q=0.9',
      'Accept-Encoding': 'gzip, deflate, br',
      'Referer': 'https://www.mercadolivre.com.br/',
      'Sec-Fetch-Dest': 'document',
      'Sec-Fetch-Mode': 'navigate',
      'Sec-Fetch-Site': 'same-site',
    },
    signal: AbortSignal.timeout(15000),
  });

  if (res.status === 403 || res.status === 429) {
    // Cookies expired — schedule refresh and throw so caller can retry
    refreshCookies().catch(() => {});
    throw new Error(`ML retornou ${res.status} — cookies expirados, atualizando...`);
  }

  const html = await res.text();

  let products = parsePreloadedState(html);
  if (!products || products.length === 0) {
    products = parseHtmlFallback(html);
  }

  if (!products || products.length === 0) {
    throw new Error('Nenhum produto encontrado no HTML retornado');
  }

  return products.slice(0, limit);
}

function injectTag(products, tag) {
  if (!tag) return products;
  return products.map(p => ({ ...p, url: `${p.url}?tag=${encodeURIComponent(tag)}` }));
}

// POST /search
app.post('/search', async (req, res) => {
  const { q, limit = 5, tag } = req.body;
  if (!q?.trim()) return res.status(400).json({ error: 'Parâmetro q obrigatório' });

  const term = q.trim();
  const cacheKey = `ml:search:${term.toLowerCase()}:${limit}`;

  try {
    const cached = await redis.get(cacheKey);
    if (cached) {
      return res.json({ products: injectTag(JSON.parse(cached), tag), cached: true });
    }
  } catch (_) {}

  try {
    let products = await searchML(term, limit);

    // If 403, retry once after a short wait (cookies should be refreshing)
    if (!products) {
      await new Promise(r => setTimeout(r, 5000));
      products = await searchML(term, limit);
    }

    await redis.setex(cacheKey, CACHE_TTL, JSON.stringify(products)).catch(() => {});
    return res.json({ products: injectTag(products, tag), cached: false });
  } catch (e) {
    return res.status(500).json({ error: e.message, products: [] });
  }
});

// POST /refresh-cookies — manual trigger
app.post('/refresh-cookies', async (req, res) => {
  const result = await refreshCookies();
  if (result) return res.json({ ok: true, message: 'Cookies atualizados' });
  return res.status(500).json({ ok: false, message: 'Falha ao atualizar cookies' });
});

// Auto-refresh cookies every 6 hours
setInterval(() => {
  refreshCookies().catch(e => console.error('[cron] cookie refresh error:', e.message));
}, 6 * 60 * 60 * 1000);

// Warm up cookies on startup
setTimeout(() => {
  getCookies().catch(e => console.error('[startup] cookie warm-up error:', e.message));
}, 3000);

const PORT = process.env.PORT || 3000;
app.listen(PORT, () => console.log(`ML Scraper rodando na porta ${PORT}`));