Here’s an overview of the nodes:
You can pay attention to only the two in the middle – Scrape Page and Extract Content.
- Set URL is just a prep node to provide the URL to be fetched. Replace this with whatever other input source you wish.
- Send a message is just an example way to see the result of the scraping.
Here’s the complete json for the two nodes in the middle (copy and paste this to n8n):
{
"nodes": [
{
"parameters": {
"url": "={{ $json.url }}",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/0.0.0.0 Safari/537.36"
},
{
"name": "Accept",
"value": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
},
{
"name": "Accept-Language",
"value": "en-US,en;q=0.9"
},
{
"name": "Accept-Encoding",
"value": "gzip, deflate, br"
},
{
"name": "Cache-Control",
"value": "no-cache"
},
{
"name": "Pragma",
"value": "no-cache"
}
]
},
"options": {
"response": {
"response": {
"neverError": true
}
},
"timeout": 20000
}
},
"id": "3bc58f1d-4a4b-4a73-be78-aaa0e810b4f5",
"name": "Scrape Page",
"type": "n8n-nodes-base.httpRequest",
"position": [
288,
208
],
"typeVersion": 4.2
},
{
"parameters": {
"jsCode": "// Extract content from HTML - handle all response types safely\nconst url = $('Set URL').item.json.url;\nlet html = '';\nlet statusCode = 'Unknown';\n\n// Filters through html content and returns only the main content part\nfunction extractMainHtml(html) {\n if (!html) return '';\n\n let cleaned = html\n .replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, '')\n .replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, '')\n .replace(/<!--[\\s\\S]*?-->/g, '')\n .replace(/<noscript[^>]*>[\\s\\S]*?<\\/noscript>/gi, '')\n\n // Remove common page chrome\n .replace(/<header[^>]*>[\\s\\S]*?<\\/header>/gi, '')\n .replace(/<nav[^>]*>[\\s\\S]*?<\\/nav>/gi, '')\n .replace(/<footer[^>]*>[\\s\\S]*?<\\/footer>/gi, '')\n .replace(/<aside[^>]*>[\\s\\S]*?<\\/aside>/gi, '')\n\n // Remove common boilerplate by class/id names\n .replace(/<[^>]+(?:id|class)=[\"'][^\"']*(?:nav|navbar|menu|footer|header|sidebar|cookie|popup|modal|subscribe|newsletter|breadcrumb|share|social|related|recommend|advert|ad-|ads)[^\"']*[\"'][^>]*>[\\s\\S]*?<\\/[^>]+>/gi, '');\n\n const candidates = [\n /<article[^>]*>[\\s\\S]*?<\\/article>/i,\n /<main[^>]*>[\\s\\S]*?<\\/main>/i,\n /<[^>]+role=[\"']main[\"'][^>]*>[\\s\\S]*?<\\/[^>]+>/i,\n /<[^>]+(?:id|class)=[\"'][^\"']*(?:article|post-content|entry-content|content-body|article-body|story-body|main-content|page-content)[^\"']*[\"'][^>]*>[\\s\\S]*?<\\/[^>]+>/i\n ];\n\n for (const pattern of candidates) {\n const match = cleaned.match(pattern);\n if (match && match[0]) {\n return match[0];\n }\n }\n\n return cleaned;\n}\n\n// Handle different response formats\nif (typeof $json === 'string') {\n html = $json;\n statusCode = 200;\n} else if ($json && $json.body) {\n html = $json.body;\n statusCode = $json.statusCode || 200;\n} else if ($json && typeof $json === 'object') {\n const possibleHtml = Object.values($json).find(val =>\n typeof val === 'string' &&\n (val.includes('<html') || val.includes('<div') || val.length > 100)\n );\n\n html = possibleHtml || '';\n statusCode = $json.statusCode || 'Unknown';\n}\n\nconsole.log('Processing URL:', url);\nconsole.log('Response type:', typeof $json);\nconsole.log('HTML length:', html ? html.length : 0);\nconsole.log('Status code:', statusCode);\n\nfunction extractTitle(html) {\n if (!html) return 'No Title';\n\n const titlePatterns = [\n /<title[^>]*>([^<]+)<\\/title>/i,\n /<meta[^>]*property=[\"']og:title[\"'][^>]*content=[\"']([^\"']+)[\"']/i,\n /<h1[^>]*>([^<]+)<\\/h1>/i\n ];\n\n for (const pattern of titlePatterns) {\n const match = html.match(pattern);\n if (match && match[1]) {\n return match[1].trim().replace(/\\s+/g, ' ');\n }\n }\n\n return 'Untitled Page';\n}\n\nfunction extractContent(html) {\n if (!html) return 'No content available';\n\n let content = extractMainHtml(html);\n\n content = content\n .replace(/<h[1-6][^>]*>([^<]+)<\\/h[1-6]>/gi, '\\n\\n## $1\\n\\n')\n .replace(/<p[^>]*>([^<]+)<\\/p>/gi, '\\n\\n$1\\n\\n')\n .replace(/<br\\s*\\/?>/gi, '\\n')\n .replace(/<div[^>]*>([^<]*)<\\/div>/gi, '\\n$1\\n')\n .replace(/<a[^>]*href=[\"']([^\"']*)[\"'][^>]*>([^<]+)<\\/a>/gi, '$2 [$1]')\n .replace(/<li[^>]*>([^<]+)<\\/li>/gi, '\\n• $1')\n .replace(/<[^>]+>/g, ' ')\n .replace(/[ \\t]+/g, ' ')\n .replace(/\\n[ \\t]+/g, '\\n')\n .replace(/[ \\t]+\\n/g, '\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n\n return content || 'No readable content found';\n}\n\nconst title = extractTitle(html);\nconst content = extractContent(html);\nconst wordCount = content.split(/\\s+/).filter(w => w.length > 0).length;\nconst hasJavaScript = html.includes('<script');\nconst timestamp = new Date().toISOString();\nconst success = html.length > 0;\n\nif (!success) {\n return {\n title: 'Scraping Failed',\n formatted: `# Scraping Failed\\n\\n**URL:** ${url}\\n**Error:** No content retrieved\\n**Status Code:** ${statusCode}\\n**Timestamp:** ${timestamp}\\n\\nThe page may be JavaScript-dependent or blocking automated requests.`\n };\n}\n\nconst formatted =\n content && content !== 'No readable content found'\n ? `# ${title}\\n\\n${content}`\n : `# ${title}\\n\\n*No readable content could be extracted from this page.*\\n\\n**Possible reasons:**\\n- Page content is loaded dynamically with JavaScript\\n- Site has anti-bot protection\\n- Content is behind authentication\\n- Page structure is complex or non-standard`;\n\nreturn {\n title,\n formatted\n};"
},
"id": "7b4d8225-9835-4ced-9f82-efd2617ace42",
"name": "Extract Content",
"type": "n8n-nodes-base.code",
"position": [
512,
208
],
"typeVersion": 2
}
],
"connections": {
"Scrape Page": {
"main": [
[
{
"node": "Extract Content",
"type": "main",
"index": 0
}
]
]
},
"Extract Content": {
"main": [
[]
]
}
},
"pinData": {},
"meta": {
"templateCredsSetupCompleted": true,
"instanceId": "4e6a6252260ef159637e3210177601ab4b7dce62704953ce56755785ae43a4f7"
}
}That’s all. Pretty simple. Works every time.