萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 腳本程式專題 >> JavaScript

Node.js + PhantomJS 爬蟲:資源下載、CSS 圖片提取與現代替代方案

架構概覽

┌─────────────┐     spawn      ┌──────────────┐
│ PhantomJS   │ ─────────────▶ │ Node.js      │
│ (down.js)   │   URL 列表      │ (downHtml.js)│
└─────────────┘                └──────────────┘
      │                              │
      ▼                              ▼
  渲染頁面                      下載資源
  收集所有請求 URL                ├─ HTML/CSS/JS/圖片
  透過 stdout 傳給 Node           └─ 解析 CSS url() 下載圖片

PhantomJS 端(`down.js`)

const page = require('webpage').create();
const system = require('system');
const { spawn } = require('child_process');

if (system.args.length < 2) {
  console.log('Usage: phantomjs down.js ');
  phantom.exit(1);
}

const urls = [];
page.onResourceReceived = res => {
  if (res.stage === 'start') urls.push(res.url);
};

page.open(system.args[1], status => {
  if (status !== 'success') {
    console.error('FAIL to load:', system.args[1]);
    phantom.exit(1);
  }
  console.log(`Captured ${urls.length} resources`);
  const child = spawn('node', ['downHtml.js', urls.join(',')]);
  child.stdout.on('data', d => console.log(d.toString()));
  child.stderr.on('data', d => console.error(d.toString()));
  child.on('exit', () => phantom.exit());
});

Node.js 端(`downHtml.js`)

const fs = require('fs');
const http = require('http');
const https = require('https');
const path = require('path');
const { URL } = require('url');

const CWD = process.cwd();
const urls = process.argv[2].split(',');
const downloaded = new Set();
const dirCache = new Set();

function mkdirp(dir) {
  if (dirCache.has(dir)) return Promise.resolve();
  return fs.promises.mkdir(dir, { recursive: true }).then(() => dirCache.add(dir));
}

function downloadFile(fileUrl, destPath) {
  return new Promise((resolve, reject) => {
    const client = fileUrl.startsWith('https:') ? https : http;
    const req = client.get(fileUrl, res => {
      if (res.statusCode >= 400) return reject(new Error(`${res.statusCode} ${fileUrl}`));
      const stream = fs.createWriteStream(destPath);
      res.pipe(stream);
      stream.on('finish', resolve);
      stream.on('error', reject);
    });
    req.on('error', reject);
    req.setTimeout(30000, () => req.destroy(new Error('Timeout')));
  });
}

async function processCss(cssUrl) {
  const cssPath = path.join(CWD, new URL(cssUrl).hostname, new URL(cssUrl).pathname);
  await mkdirp(path.dirname(cssPath));
  await downloadFile(cssUrl, cssPath);
  const css = await fs.promises.readFile(cssPath, 'utf8');
  const urlRegex = /url\(['"]?([^)'"]+)['"]?\)/g;
  let match;
  while ((match = urlRegex.exec(css)) !== null) {
    const imgUrl = new URL(match[1], cssUrl).href;
    if (downloaded.has(imgUrl)) continue;
    downloaded.add(imgUrl);
    const imgPath = path.join(CWD, new URL(imgUrl).hostname, new URL(imgUrl).pathname);
    await mkdirp(path.dirname(imgPath));
    await downloadFile(imgUrl, imgPath).catch(err => console.warn('CSS image failed:', imgUrl, err.message));
  }
}

(async () => {
  for (const url of urls) {
    if (downloaded.has(url)) continue;
    downloaded.add(url);
    try {
      const u = new URL(url);
      const dest = path.join(CWD, u.hostname, u.pathname === '/' ? 'index.html' : u.pathname);
      await mkdirp(path.dirname(dest));
      await downloadFile(url, dest);
      console.log('Downloaded:', url);
      if (url.endsWith('.css') || url.includes('.css')) await processCss(url);
    } catch (e) {
      console.error('Failed:', url, e.message);
    }
  }
})();

執行指令

phantomjs down.js https://example.com/

重大缺陷與現代替代方案

問題影響現代替代
PhantomJS **已停止維護**(2018)無 ES6+ 支援、無 Chrome DevTools Protocol、安全漏洞不修Puppeteer / Playwright (Headless Chrome/Firefox/WebKit)
無法執行現代 JS 框架渲染React/Vue/Angular SPA 內容抓不到Puppeteer `page.waitForSelector`、`page.evaluate` 完整執行 JS
CSS `url()` 解析不全漏抓 `@import`、`image-set()`、動態載入圖片Puppeteer 可攔截 `page.on('response')` 直接拿所有資源
單進程、無並發控制大量資源下載極慢Node `p-limit` / `async-pool` 並發下載

Puppeteer 現代重寫(約 30 行搞定全站資源下載)

const puppeteer = require('puppeteer');
const fs = require('fs').promises;
const path = require('path');
const { URL } = require('url');
const pLimit = require('p-limit'); // npm i p-limit

async function downloadSite(targetUrl, outDir = './download') {
  const browser = await puppeteer.launch({ headless: 'new' });
  const page = await browser.newPage();
  const resources = new Map(); // url → {buffer, contentType}

  page.on('response', async res => {
    const url = res.url();
    if (resources.has(url)) return;
    try {
      const buffer = await res.buffer();
      resources.set(url, { buffer, contentType: res.headers()['content-type'] });
    } catch (e) { /* ignore */ }
  });

  await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 60000 });
  await browser.close();

  const limit = pLimit(8);
  await Promise.all([...resources.entries()].map(([url, { buffer, contentType }]) =>
    limit(async () => {
      const u = new URL(url);
      const rel = u.pathname === '/' ? 'index.html' : u.pathname.replace(/^\//, '');
      const dest = path.join(outDir, u.hostname, rel);
      await fs.mkdir(path.dirname(dest), { recursive: true });
      await fs.writeFile(dest, buffer);
      // CSS 進一步解析圖片(可選)
      if (contentType?.includes('css')) {
        const css = buffer.toString();
        const regex = /url\(['"]?([^)'"]+)['"]?\)/g;
        for (const m of css.matchAll(regex)) {
          const imgUrl = new URL(m[1], url).href;
          if (!resources.has(imgUrl)) {
            // 這裡可再發請求下載,略
          }
        }
      }
    })
  ));
  console.log(`Done. ${resources.size} files saved to ${outDir}`);
}

downloadSite('https://example.com');

結語

PhantomJS 時代已終結。現代爬蟲請直接使用 **Puppeteer** 或 **Playwright**:完整瀏覽器環境、原生支援現代 JS、強大的網路攔截、TypeScript 一級支援、活躍維護。上述代碼可直接用於生產環境的靜態站點鏡像、離線閱讀、SEO 快照等場景。

copyright © 萬盛學電腦網 all rights reserved