┌─────────────┐ spawn ┌──────────────┐
│ PhantomJS │ ─────────────▶ │ Node.js │
│ (down.js) │ URL 列表 │ (downHtml.js)│
└─────────────┘ └──────────────┘
│ │
▼ ▼
渲染頁面 下載資源
收集所有請求 URL ├─ HTML/CSS/JS/圖片
透過 stdout 傳給 Node └─ 解析 CSS url() 下載圖片
const page = require('webpage').create();
const system = require('system');
const { spawn } = require('child_process');
if (system.args.length < 2) {
console.log('Usage: phantomjs down.js ');
phantom.exit(1);
}
const urls = [];
page.onResourceReceived = res => {
if (res.stage === 'start') urls.push(res.url);
};
page.open(system.args[1], status => {
if (status !== 'success') {
console.error('FAIL to load:', system.args[1]);
phantom.exit(1);
}
console.log(`Captured ${urls.length} resources`);
const child = spawn('node', ['downHtml.js', urls.join(',')]);
child.stdout.on('data', d => console.log(d.toString()));
child.stderr.on('data', d => console.error(d.toString()));
child.on('exit', () => phantom.exit());
});
const fs = require('fs');
const http = require('http');
const https = require('https');
const path = require('path');
const { URL } = require('url');
const CWD = process.cwd();
const urls = process.argv[2].split(',');
const downloaded = new Set();
const dirCache = new Set();
function mkdirp(dir) {
if (dirCache.has(dir)) return Promise.resolve();
return fs.promises.mkdir(dir, { recursive: true }).then(() => dirCache.add(dir));
}
function downloadFile(fileUrl, destPath) {
return new Promise((resolve, reject) => {
const client = fileUrl.startsWith('https:') ? https : http;
const req = client.get(fileUrl, res => {
if (res.statusCode >= 400) return reject(new Error(`${res.statusCode} ${fileUrl}`));
const stream = fs.createWriteStream(destPath);
res.pipe(stream);
stream.on('finish', resolve);
stream.on('error', reject);
});
req.on('error', reject);
req.setTimeout(30000, () => req.destroy(new Error('Timeout')));
});
}
async function processCss(cssUrl) {
const cssPath = path.join(CWD, new URL(cssUrl).hostname, new URL(cssUrl).pathname);
await mkdirp(path.dirname(cssPath));
await downloadFile(cssUrl, cssPath);
const css = await fs.promises.readFile(cssPath, 'utf8');
const urlRegex = /url\(['"]?([^)'"]+)['"]?\)/g;
let match;
while ((match = urlRegex.exec(css)) !== null) {
const imgUrl = new URL(match[1], cssUrl).href;
if (downloaded.has(imgUrl)) continue;
downloaded.add(imgUrl);
const imgPath = path.join(CWD, new URL(imgUrl).hostname, new URL(imgUrl).pathname);
await mkdirp(path.dirname(imgPath));
await downloadFile(imgUrl, imgPath).catch(err => console.warn('CSS image failed:', imgUrl, err.message));
}
}
(async () => {
for (const url of urls) {
if (downloaded.has(url)) continue;
downloaded.add(url);
try {
const u = new URL(url);
const dest = path.join(CWD, u.hostname, u.pathname === '/' ? 'index.html' : u.pathname);
await mkdirp(path.dirname(dest));
await downloadFile(url, dest);
console.log('Downloaded:', url);
if (url.endsWith('.css') || url.includes('.css')) await processCss(url);
} catch (e) {
console.error('Failed:', url, e.message);
}
}
})();
phantomjs down.js https://example.com/
| 問題 | 影響 | 現代替代 |
|---|---|---|
| PhantomJS **已停止維護**(2018) | 無 ES6+ 支援、無 Chrome DevTools Protocol、安全漏洞不修 | Puppeteer / Playwright (Headless Chrome/Firefox/WebKit) |
| 無法執行現代 JS 框架渲染 | React/Vue/Angular SPA 內容抓不到 | Puppeteer `page.waitForSelector`、`page.evaluate` 完整執行 JS |
| CSS `url()` 解析不全 | 漏抓 `@import`、`image-set()`、動態載入圖片 | Puppeteer 可攔截 `page.on('response')` 直接拿所有資源 |
| 單進程、無並發控制 | 大量資源下載極慢 | Node `p-limit` / `async-pool` 並發下載 |
const puppeteer = require('puppeteer');
const fs = require('fs').promises;
const path = require('path');
const { URL } = require('url');
const pLimit = require('p-limit'); // npm i p-limit
async function downloadSite(targetUrl, outDir = './download') {
const browser = await puppeteer.launch({ headless: 'new' });
const page = await browser.newPage();
const resources = new Map(); // url → {buffer, contentType}
page.on('response', async res => {
const url = res.url();
if (resources.has(url)) return;
try {
const buffer = await res.buffer();
resources.set(url, { buffer, contentType: res.headers()['content-type'] });
} catch (e) { /* ignore */ }
});
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 60000 });
await browser.close();
const limit = pLimit(8);
await Promise.all([...resources.entries()].map(([url, { buffer, contentType }]) =>
limit(async () => {
const u = new URL(url);
const rel = u.pathname === '/' ? 'index.html' : u.pathname.replace(/^\//, '');
const dest = path.join(outDir, u.hostname, rel);
await fs.mkdir(path.dirname(dest), { recursive: true });
await fs.writeFile(dest, buffer);
// CSS 進一步解析圖片(可選)
if (contentType?.includes('css')) {
const css = buffer.toString();
const regex = /url\(['"]?([^)'"]+)['"]?\)/g;
for (const m of css.matchAll(regex)) {
const imgUrl = new URL(m[1], url).href;
if (!resources.has(imgUrl)) {
// 這裡可再發請求下載,略
}
}
}
})
));
console.log(`Done. ${resources.size} files saved to ${outDir}`);
}
downloadSite('https://example.com');
PhantomJS 時代已終結。現代爬蟲請直接使用 **Puppeteer** 或 **Playwright**:完整瀏覽器環境、原生支援現代 JS、強大的網路攔截、TypeScript 一級支援、活躍維護。上述代碼可直接用於生產環境的靜態站點鏡像、離線閱讀、SEO 快照等場景。