大文件上传解决方案-基于Node.js的分片与断点续传实践

const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
const chunk = blobSlice.call(file, start, end); // 返回指定范围的 Blob

const fileReader = new FileReader();
fileReader.readAsArrayBuffer(blob); // 读取为 ArrayBuffer
fileReader.onload = (e) => {
  console.log(e.target.result); // 读取结果
};

const formData = new FormData();
formData.append('chunk', new Blob([chunkData])); // 文件分片
formData.append('index', 1); // 分片索引
formData.append('fileHash', 'xxx'); // 文件哈希

import SparkMD5 from 'spark-md5';

/**
 * 将目标文件分片并计算文件 Hash
 * @param {File} targetFile 目标上传文件
 * @param {number} baseChunkSize 分片大小（单位 MB）
 * @returns {chunkList: ArrayBuffer[], fileHash: string}
 */
async function sliceFile(targetFile, baseChunkSize = 1) {
  return new Promise((resolve, reject) => {
    // 兼容不同浏览器的分片方法
    const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;

    const chunkSize = baseChunkSize * 1024 * 1024;
    const targetChunkCount = Math.ceil(targetFile.size / chunkSize);
    let currentChunkCount = 0;
    const chunkList = [];

    const spark = new SparkMD5.ArrayBuffer();
    const fileReader = new FileReader();

    fileReader.onload = (e) => {
      const curChunk = e.target.result;
      spark.append(curChunk); // 追加到 Hash 计算
      currentChunkCount++;
      chunkList.push(curChunk);

      if (currentChunkCount >= targetChunkCount) {
        // 全部读取完成，获取文件 Hash
        const fileHash = spark.end();
        resolve({ chunkList, fileHash });
      } else {
        loadNext();
      }
    };

    fileReader.onerror = () => reject(null);

    const loadNext = () => {
      const start = chunkSize * currentChunkCount;
      const end = Math.min(start + chunkSize, targetFile.size);
      fileReader.readAsArrayBuffer(blobSlice.call(targetFile, start, end));
    };

    loadNext();
  });
}

async function uploadFile(file, baseChunkSize, uploadUrl, verifyUrl, mergeUrl, progressCallback) {
  // 1. 分片并计算 Hash
  const { chunkList, fileHash } = await sliceFile(file, baseChunkSize);

  let neededChunkList = [];
  let progress = 0;

  // 2. 验证文件上传状态
  if (verifyUrl) {
    const { data } = await requestInstance.post(verifyUrl, {
      fileHash,
      totalCount: chunkList.length,
      extname: '.' + file.name.split('.').pop(),
    });

    const { neededFileList, message } = data;

    // 秒传成功
    if (!neededFileList.length) {
      progressCallback(100);
      return;
    }

    neededChunkList = neededFileList;
  }

  // 3. 计算断点续传的初始进度
  progress = ((chunkList.length - neededChunkList.length) / chunkList.length) * 100;

  // 4. 上传分片
  if (chunkList.length) {
    const requestList = chunkList.map(async (chunk, index) => {
      if (neededChunkList.includes(index + 1)) {
        await uploadChunk(chunk, index + 1, fileHash, uploadUrl);

        // 更新进度
        progress += Math.ceil(100 / chunkList.length);
        if (progress >= 100) progress = 100;
        progressCallback(progress);
      }
    });

    Promise.all(requestList).then(() => {
      // 5. 通知服务端合并分片
      requestInstance.post(mergeUrl, {
        fileHash,
        extname: '.mp4'
      });
    });
  }
}

/**
 * 上传单个分片
 */
async function uploadChunk(chunk, index, fileHash, uploadUrl) {
  const formData = new FormData();
  // ArrayBuffer 需要转为 Blob
  formData.append('chunk', new Blob([chunk]));
  formData.append('index', index);
  formData.append('fileHash', fileHash);
  return requestInstance.post(uploadUrl, formData);
}

const fs = require('fs');
const path = require('path');

uploadChunk(chunk, chunkInfo) {
  const { fileHash, index } = chunkInfo;
  const dirPath = path.join(__dirname, 'uploadedFiles/chunkFile', fileHash);
  const chunkPath = path.join(dirPath, `chunk-${index}`);

  // 检查文件夹是否存在
  if (!fs.existsSync(dirPath)) {
    fs.mkdirSync(dirPath, { recursive: true });
  }

  // 检查分片是否已存在
  if (fs.existsSync(chunkPath)) {
    return;
  }

  // 写入分片文件
  fs.writeFileSync(chunkPath, chunk.buffer);
}