Web Scraping met n8n: Complete Gids voor Playwright & Puppeteer in 2026

In het tijdperk van JavaScript frameworks, single-page applications en dynamische content is traditionele web scraping niet meer voldoende. Met n8n's integratie van Playwright en Puppeteer kun je echter elke website scrapen - ongeacht hoe complex of beschermd.

Waarom Browser Automation voor Web Scraping?

Traditionele scraping tools zoals BeautifulSoup of zelfs n8n's HTTP Request node falen bij moderne websites omdat ze:

Playwright vs Puppeteer: Welke Kiezen?

Installatie & Setup in n8n

Optie 1: Puppeteer Community Node

# Voor n8n versie 0.187+
# Ga naar Settings → Community Nodes
# Installeer: n8n-nodes-puppeteer

# Of via npm voor self-hosted:
npm install n8n-nodes-puppeteer

# Docker users, voeg toe aan je docker-compose.yml:
environment:
  - N8N_COMMUNITY_PACKAGES=n8n-nodes-puppeteer

Optie 2: Playwright via Code Node

// In een Code node:
const playwright = require('playwright');

// Launch browser
const browser = await playwright.chromium.launch({
  headless: true,  // false voor debugging
  args: ['--no-sandbox']  // Voor Docker environments
});

const page = await browser.newPage();

// Ga naar website
await page.goto('https://example.com', {
  waitUntil: 'networkidle'  // Wacht tot alle requests klaar zijn
});

// Scrape data
const data = await page.evaluate(() => {
  return {
    title: document.title,
    products: Array.from(document.querySelectorAll('.product')).map(el => ({
      name: el.querySelector('.name')?.textContent,
      price: el.querySelector('.price')?.textContent,
      image: el.querySelector('img')?.src
    }))
  };
});

await browser.close();
return data;

Praktijkvoorbeeld 1: E-commerce Price Monitoring

Automatisch prijzen monitoren van concurrenten met dynamic content loading:

// Complete Puppeteer price monitoring script
const puppeteer = require('puppeteer');

async function scrapePrices() {
  const browser = await puppeteer.launch({
    headless: true,
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-blink-features=AutomationControlled'  // Anti-detection
    ]
  });

  const page = await browser.newPage();
  
  // Set user agent voor anti-bot bypass
  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
  
  // Extra headers voor authenticiteit
  await page.setExtraHTTPHeaders({
    'Accept-Language': 'nl-NL,nl;q=0.9'
  });

  try {
    await page.goto('https://www.bol.com/nl/nl/s/?searchtext=laptop', {
      waitUntil: 'networkidle2',
      timeout: 60000
    });

    // Wacht tot producten geladen zijn
    await page.waitForSelector('[data-test=\"product-title\"]', {
      timeout: 30000
    });

    // Scroll voor lazy loading
    await autoScroll(page);

    // Extract product data
    const products = await page.evaluate(() => {
      const items = document.querySelectorAll('[data-test=\"product-item\"]');
      return Array.from(items).map(item => {
        const title = item.querySelector('[data-test=\"product-title\"]')?.textContent?.trim();
        const priceWhole = item.querySelector('.promo-price__amount--whole')?.textContent;
        const priceFraction = item.querySelector('.promo-price__amount--fraction')?.textContent;
        const price = priceWhole && priceFraction ? 
          parseFloat(`${priceWhole}.${priceFraction}`) : null;
        const image = item.querySelector('img')?.src;
        const rating = item.querySelector('[data-test=\"rating-stars\"]')?.getAttribute('title');
        const seller = item.querySelector('[data-test=\"seller-name\"]')?.textContent?.trim();
        
        return {
          title,
          price,
          image,
          rating,
          seller,
          scraped_at: new Date().toISOString()
        };
      }).filter(item => item.title && item.price);
    });

    await browser.close();
    return products;

  } catch (error) {
    console.error('Scraping failed:', error);
    await browser.close();
    throw error;
  }
}

// Auto-scroll functie voor infinite scroll
async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve) => {
      let totalHeight = 0;
      const distance = 100;
      const timer = setInterval(() => {
        const scrollHeight = document.body.scrollHeight;
        window.scrollBy(0, distance);
        totalHeight += distance;

        if(totalHeight >= scrollHeight - window.innerHeight){
          clearInterval(timer);
          resolve();
        }
      }, 100);
    });
  });
}

Praktijkvoorbeeld 2: LinkedIn Lead Generation

Automatisch LinkedIn profielen scrapen met login handling:

// Playwright script voor LinkedIn scraping
const { chromium } = require('playwright');

async function scrapeLinkedIn(email, password, searchQuery) {
  const browser = await chromium.launch({
    headless: false,  // LinkedIn detecteert headless vaak
    slowMo: 50       // Menselijke snelheid simuleren
  });

  const context = await browser.newContext({
    viewport: { width: 1920, height: 1080 },
    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)...'
  });

  const page = await context.newPage();

  // Login flow
  await page.goto('https://www.linkedin.com/login');
  await page.fill('#username', email);
  await page.fill('#password', password);
  await page.click('[type=\"submit\"]');
  
  // Wacht op redirect na login
  await page.waitForURL('**/feed/**', { timeout: 60000 });
  
  // Ga naar search
  await page.goto(`https://www.linkedin.com/search/results/people/?keywords=${searchQuery}`);
  
  // Wacht op resultaten
  await page.waitForSelector('.search-results-container');
  
  // Extract profielen
  const profiles = await page.evaluate(() => {
    const results = document.querySelectorAll('.entity-result');
    return Array.from(results).map(result => {
      const name = result.querySelector('.entity-result__title-text')?.textContent?.trim();
      const title = result.querySelector('.entity-result__primary-subtitle')?.textContent?.trim();
      const location = result.querySelector('.entity-result__secondary-subtitle')?.textContent?.trim();
      const link = result.querySelector('.app-aware-link')?.href;
      
      return { name, title, location, link };
    });
  });

  await browser.close();
  return profiles;
}

Anti-Bot Detection & Stealth Technieken

Moderne websites gebruiken geavanceerde bot-detectie. Hier zijn technieken om deze te omzeilen:

// Puppeteer Stealth Plugin
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());

const browser = await puppeteer.launch({
  headless: false,  // Of 'new' voor Headless Chrome v2
  args: [
    '--disable-blink-features=AutomationControlled',
    '--disable-features=IsolateOrigins,site-per-process',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',
    '--disable-accelerated-2d-canvas',
    '--no-first-run',
    '--no-zygote',
    '--single-process',
    '--disable-gpu'
  ]
});

// Remove navigator.webdriver flag
await page.evaluateOnNewDocument(() => {
  Object.defineProperty(navigator, 'webdriver', {
    get: () => undefined
  });
});

// Randomize viewport
await page.setViewport({
  width: 1920 + Math.floor(Math.random() * 100),
  height: 1080 + Math.floor(Math.random() * 100)
});

// Human-like mouse movements
await page.mouse.move(100, 200);
await page.waitForTimeout(1000 + Math.random() * 2000);

Performance Optimalisatie

Data Extractie Patterns

// Geavanceerde data extractie technieken
// 1. Wait for AJAX content
await page.waitForFunction(
  () => document.querySelectorAll('.product').length > 10,
  { timeout: 30000 }
);

// 2. Extract met error handling
const data = await page.evaluate(() => {
  const extractText = (selector, parent = document) => {
    const element = parent.querySelector(selector);
    return element ? element.textContent.trim() : null;
  };

  const extractAttr = (selector, attr, parent = document) => {
    const element = parent.querySelector(selector);
    return element ? element.getAttribute(attr) : null;
  };

  const products = [];
  document.querySelectorAll('.product-card').forEach(card => {
    try {
      products.push({
        name: extractText('.product-name', card),
        price: extractText('.price', card)?.replace(/[^0-9.,]/g, ''),
        image: extractAttr('img', 'src', card),
        link: extractAttr('a', 'href', card),
        inStock: !card.classList.contains('out-of-stock'),
        reviews: parseInt(extractText('.review-count', card)) || 0
      });
    } catch (e) {
      console.error('Failed to extract product:', e);
    }
  });
  return products;
});

// 3. Screenshot voor debugging
await page.screenshot({ 
  path: 'debug.png', 
  fullPage: true 
});

// 4. Save als JSON
const fs = require('fs');
fs.writeFileSync('products.json', JSON.stringify(data, null, 2));

n8n Workflow Integratie

Complete n8n workflow voor automated scraping:

Troubleshooting & Common Issues

Cloud & Scaling Oplossingen

Voor productie-scale scraping:

Legal & Ethische Overwegingen

Conclusie & Best Practices

Web scraping met n8n, Playwright en Puppeteer opent een wereld van mogelijkheden voor data extractie en automatisering. Met de juiste technieken kun je zelfs de meest complexe websites scrapen.

Begin klein met de n8n-nodes-puppeteer community node, experimenteer met verschillende sites, en bouw geleidelijk complexere workflows. De combinatie van n8n's visual workflow builder met de kracht van browser automation maakt web scraping toegankelijk voor iedereen.