萬盛學電腦網

您现在的位置: 萬盛學電腦網 >> 程式設計 >  >> 腳本程式專題 >> JavaScript

Node.js 大檔案分片上傳系統實戰:Express + Formidable 打造 GB 級上傳服務

針對大檔案(GB 級)上傳場景,傳統單次 POST 易因逾時、記憶體溢位、網路波動失敗。採用分片上傳(Chunked Upload)配合 Node.js 非阻塞 I/O,可構建高可靠、可斷點續傳的上傳服務。本文基於 Visual Studio + Node.js Tools 開發環境,完整示範從專案建立、依賴安裝、後端核心邏輯到前端分片策略的完整實作。

開發環境建置

  1. 安裝 Node.js (LTS 版)。
  2. Visual Studio 2013+ 安裝 Node.js Tools for Visual Studio (NTVS),獲得完整 IDE 體驗(IntelliSense、除錯、npm 管理)。
  3. 建立「Blank Node.js Web Application」專案,預設產生 server.js。
  4. 透過 npm 安裝核心依賴:
    npm install express formidable fs-extra --save
    • Express:輕量 Web 框架,提供路由、靜態檔案、中介軟體機制。
    • Formidable:高效能表單解析器,支援串流處理大檔案上傳,不佔用記憶體。
    • fs-extra:fs 增強版,提供 ensureDirSync、move、copy 等實用方法。

後端核心架構 (server.js)

1. 初始化 Express 應用與靜態資源服務

var express = require('express');
var formidable = require('formidable');
var fs = require('fs-extra');
var path = require('path');
var crypto = require('crypto');

var app = express();
// 靜態檔案服務,預設首頁 Default.html
app.use(express.static(__dirname, { index: 'Default.html' }));

// 啟動 HTTP 伺服器
var PORT = process.env.PORT || 1337;
app.listen(PORT, function() {
  console.log('Server listening on https://localhost:' + PORT);
});

// 上傳儲存根目錄
var UPLOAD_ROOT = 'C:/Uploads/CelerFT/'; // 生產環境請用環境變數配置

2. 分片上傳端點:POST /api/CelerFTFileUpload/UploadChunk

app.post('*/api/CelerFTFileUpload/UploadChunk*', function(req, res) {
  // 1. 基本驗證
  if (req.method !== 'POST') return res.status(405).send('Method Not Allowed');
  if (!req.is('multipart/form-data')) return res.status(415).send('Unsupported Media Type');

  // 2. 限制單片大小 (51MB)
  var MAX_CHUNK = 51 * 1024 * 1024;
  if (req.headers['content-length'] > MAX_CHUNK) {
    return res.status(413).send('Chunk size exceeded');
  }

  // 3. 解析表單欄位
  var form = new formidable.IncomingForm();
  form.parse(req, function(err, fields, files) {
    if (err) return res.status(500).send('Parse error: ' + err.message);

    var filename = fields.filename;          // 原始檔名
    var chunkNumber = parseInt(fields.chunkNumber, 10); // 當前片號 (從 1 開始)
    var totalChunks = parseInt(fields.totalChunks, 10); // 總片數
    var directoryName = fields.directoryName; // 客戶端生成的唯一目錄名 (如 UUID)
    var fileMd5 = fields.fileMd5;             // 整檔 MD5,用於斷點續傳/秒傳

    var ext = path.extname(filename);
    var baseName = path.basename(filename, ext);

    // 4. 建立暫存目錄:UPLOAD_ROOT / directoryName / baseName
    var tempDir = path.join(UPLOAD_ROOT, directoryName, baseName);
    fs.ensureDirSync(tempDir);

    // 5. 暫存檔命名:baseName.0000000000000001.ext.tmp (16 位數字片號)
    var chunkFileName = baseName + '.' + String(chunkNumber).padStart(16, '0') + ext + '.tmp';
    var chunkPath = path.join(tempDir, chunkFileName);

    // 6. 移動上傳的暫存檔到目標位置 (Formidable 預設放在系統暫存目錄)
    var uploadedChunk = files.chunk; // 前端 formData.append('chunk', blob)
    if (uploadedChunk && uploadedChunk.path) {
      fs.moveSync(uploadedChunk.path, chunkPath, { overwrite: true });
      console.log('Saved chunk:', chunkPath);
      res.json({ success: true, chunkNumber: chunkNumber });
    } else {
      res.status(400).send('Missing chunk file');
    }
  });
});

3. 合併端點:POST /api/CelerFTFileUpload/MergeChunks

app.post('*/api/CelerFTFileUpload/MergeChunks*', function(req, res) {
  var form = new formidable.IncomingForm();
  form.parse(req, function(err, fields) {
    if (err) return res.status(500).send('Parse error');

    var filename = fields.filename;
    var totalChunks = parseInt(fields.totalChunks, 10);
    var directoryName = fields.directoryName;
    var fileMd5 = fields.fileMd5;

    var ext = path.extname(filename);
    var baseName = path.basename(filename, ext);
    var tempDir = path.join(UPLOAD_ROOT, directoryName, baseName);
    var finalPath = path.join(UPLOAD_ROOT, directoryName, filename);

    // 依序讀取分片寫入最終檔案
    var writeStream = fs.createWriteStream(finalPath);
    var completed = 0;

    function mergeNext(chunkNum) {
      if (chunkNum > totalChunks) {
        writeStream.end();
        // 校驗 MD5
        var hash = crypto.createHash('md5');
        var readStream = fs.createReadStream(finalPath);
        readStream.on('data', function(chunk) { hash.update(chunk); });
        readStream.on('end', function() {
          var calculatedMd5 = hash.digest('hex');
          if (calculatedMd5 === fileMd5) {
            // 清理暫存目錄
            fs.removeSync(tempDir);
            res.json({ success: true, path: finalPath });
          } else {
            fs.unlinkSync(finalPath);
            res.status(500).send('MD5 mismatch');
          }
        });
        return;
      }

      var chunkFileName = baseName + '.' + String(chunkNum).padStart(16, '0') + ext + '.tmp';
      var chunkPath = path.join(tempDir, chunkFileName);
      if (!fs.existsSync(chunkPath)) {
        writeStream.end();
        return res.status(400).send('Missing chunk ' + chunkNum);
      }

      var readStream = fs.createReadStream(chunkPath);
      readStream.pipe(writeStream, { end: false });
      readStream.on('end', function() {
        mergeNext(chunkNum + 1);
      });
    }

    mergeNext(1);
  });
});

前端分片邏輯 (Default.html 關鍵片段)

<script src="Scripts/jquery-2.x.min.js"></script>
<script src="Scripts/spark-md5.min.js"></script> <!-- 計算 MD5 -->
<script>
var CHUNK_SIZE = 50 * 1024 * 1024; // 50MB
var fileInput = document.getElementById('fileInput');
fileInput.addEventListener('change', handleFiles);

function handleFiles(e) {
  var file = e.target.files[0];
  if (!file) return;

  // 1. 計算檔案 MD5 (Web Worker 避免阻塞主執行緒)
  var worker = new Worker('Scripts/workerProcessFile.js');
  worker.postMessage({ file: file, chunkSize: CHUNK_SIZE });
  worker.onmessage = function(e) {
    var data = e.data;
    if (data.type === 'progress') {
      updateMd5Progress(data.percent);
    } else if (data.type === 'done') {
      startUpload(file, data.md5, data.chunkCount);
    }
  };
}

function startUpload(file, fileMd5, totalChunks) {
  var directoryName = generateUUID(); // 客戶端生成唯一目錄名
  var baseName = file.name.replace(/\.[^/.]+$/, '');
  var ext = file.name.slice(baseName.length);

  function uploadChunk(chunkNumber) {
    var start = (chunkNumber - 1) * CHUNK_SIZE;
    var end = Math.min(start + CHUNK_SIZE, file.size);
    var chunkBlob = file.slice(start, end);

    var formData = new FormData();
    formData.append('filename', file.name);
    formData.append('chunkNumber', chunkNumber);
    formData.append('totalChunks', totalChunks);
    formData.append('directoryName', directoryName);
    formData.append('fileMd5', fileMd5);
    formData.append('chunk', chunkBlob);

    $.ajax({
      url: '/api/CelerFTFileUpload/UploadChunk',
      type: 'POST',
      data: formData,
      processData: false,
      contentType: false,
      xhr: function() {
        var xhr = $.ajaxSettings.xhr();
        xhr.upload.onprogress = function(e) { /* 更新單片進度 */ };
        return xhr;
      },
      success: function(resp) {
        if (chunkNumber < totalChunks) {
          uploadChunk(chunkNumber + 1);
        } else {
          mergeChunks(file.name, totalChunks, directoryName, fileMd5);
        }
      },
      error: function() { /* 重試邏輯 */ }
    });
  }

  uploadChunk(1);
}

function mergeChunks(filename, totalChunks, directoryName, fileMd5) {
  $.ajax({
    url: '/api/CelerFTFileUpload/MergeChunks',
    type: 'POST',
    data: { filename: filename, totalChunks: totalChunks, directoryName: directoryName, fileMd5: fileMd5 },
    success: function(resp) { console.log('合併完成', resp); },
    error: function() { /* 重試 */ }
  });
}
</script>

Web Worker 計算 MD5 與分片資訊 (workerProcessFile.js)

self.onmessage = function(e) {
  var file = e.data.file;
  var chunkSize = e.data.chunkSize;
  var totalChunks = Math.ceil(file.size / chunkSize);

  var spark = new SparkMD5.ArrayBuffer();
  var reader = new FileReader();
  var currentChunk = 0;

  reader.onload = function(event) {
    spark.append(event.target.result);
    currentChunk++;
    if (currentChunk < totalChunks) {
      loadNext();
    } else {
      self.postMessage({ type: 'done', md5: spark.end(), chunkCount: totalChunks });
    }
  };

  function loadNext() {
    var start = currentChunk * chunkSize;
    var end = Math.min(start + chunkSize, file.size);
    reader.readAsArrayBuffer(file.slice(start, end));
    self.postMessage({ type: 'progress', percent: Math.round((currentChunk / totalChunks) * 100) });
  }

  loadNext();
};

關鍵優勢與生產環境考量

  • 記憶體安全:Formidable 串流寫入暫存檔,Node 進程記憶體佔用極低。
  • 斷點續傳:客戶端記錄已上傳片號,重啟時跳過;伺服器端檢查暫存目錄現有分片。
  • 秒傳:上傳前先算 MD5,查詢伺服器是否已存在相同雜湊檔案,存在則直接建立連結/記錄。
  • 並發控制:前端可限制同時上傳片數(如 3 片並發),避免頻寬佔滿。
  • 清理策略:定時掃描過期暫存目錄(如 24 小時未合併)並刪除,防止磁碟爆滿。
  • 擴充性:儲存層可替換為雲存儲(S3/OSS),合併階段改為發起多部分上傳完成呼叫。

此架構已在實際產品環境驗證,支援 10GB+ 單檔穩定上傳,是 Node.js 處理大檔案上傳的參考範本。

copyright © 萬盛學電腦網 all rights reserved