客户端（web）

hash.worker.js

import SparkMD5 from 'spark-md5'
onmessage = function(event) {
  getFileHash(event.data)
}

function getFileHash({file, chunkSize}) {
  console.log(file, chunkSize)
  const spark = new SparkMD5.ArrayBuffer()
  const reader = new FileReader()
  reader.readAsArrayBuffer(file)

  reader.addEventListener('loadend', () => {
    const content = reader.result
    // 抽样hash计算
    // 规则：每半个切片大小取前10个
    let i = 0

    while(chunkSize / 2 * (i + 1) + 10 < file.size) {
      spark.append(content.slice(chunkSize / 2 * i, chunkSize / 2 * i + 10))
      i++
    }

    const hash = spark.end()
    postMessage(hash)
  })

  reader.addEventListener('error', function _error(err) {
    postMessage(err)
  })
}

app.vue

<template>
  <div>
    <input type="file" @change="uploadFile">
  </div>
</template>

<script>
// import SparkMD5 from 'spark-md5'
import Worker from './hash.worker.js'

export default {
  data() {
    return {
      fileInfo: null,
      chunkSize: 100 * 1024 // 切片大小
    }
  },
  methods: {
    // input改变事件监听
    uploadFile(e) {
      const file = e.target.files[0]
      // 如果文件大小大于文件分片大小的5倍才使用分片上传
      if (file.size / this.chunkSize < 5) {
        this.sendFile(file)
        return
      }
      this.createFileMd5(file).then(async fileMd5 => {
        // 先查询服务器是否已有上传完的文件切片
        let {data} = await this.getUploadedChunks(fileMd5)
        let uploaded = data.data.length ? data.data.map(v => v.split('-')[1] - 0) : []
        // 切割文件
        const chunkArr = await this.cutBlob(fileMd5, file, uploaded)
        // 开始上传
        this.sendRequest(chunkArr, 5, this.chunkMerge)
      })
    },
    createFileMd5(file) {
      return new Promise((resolve) => {
        const worker = new Worker()

        worker.postMessage({file, chunkSize: this.chunkSize})

        worker.onmessage = event => {
          resolve(event.data)
        }
      })
    },
    // 文件分割
    cutBlob(fileHash, file, uploaded) {
      const chunkArr = [] // 所有切片缓存数组
      const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice
      const chunkNums = Math.ceil(file.size / this.chunkSize) // 切片总数

      return new Promise(resolve => {
        let startIndex = ''
        let endIndex = ''
        let contentItem = ''

        for(let i = 0; i < chunkNums; i++) {
          // 如果已上传则跳过
          if (uploaded.includes(i)) continue

          startIndex = i * this.chunkSize // 片段起点
          endIndex = (i + 1) * this.chunkSize // 片段尾点
          endIndex > file.size && (endIndex = file.size)

          // 切割文件
          contentItem = blobSlice.call(file, startIndex, endIndex)

          chunkArr.push({
            index: i,
            chunk: contentItem
          })
        }
        this.fileInfo = {
          hash: fileHash,
          total: chunkNums,
          name: file.name,
          size: file.size
        }
        resolve(chunkArr)
      })
    },
    // 请求并发处理
    sendRequest(arr, max = 6, callback) {
      let fetchArr = []

      let toFetch = () => {
        if (!arr.length) {
          return Promise.resolve()
        }

        const chunkItem = arr.shift()

        const it = this.sendChunk(chunkItem)
        it.then(() => {
          // 成功从任务队列中移除
          fetchArr.splice(fetchArr.indexOf(it), 1)
        }, err => {
          // 如果失败则重新放入总队列中
          arr.unshift(chunkItem)
          console.log(err)
        })
        fetchArr.push(it)

        let p = Promise.resolve()
        if (fetchArr.length >= max) {
          p = Promise.race(fetchArr)
        }

        return p.then(() => toFetch())
      }

      toFetch().then(() => {
        Promise.all(fetchArr).then(() => {
          callback()
        })
      }, err => {
        console.log(err)
      })
    },
    // 请求已上传文件
    getUploadedChunks(hash) {
      return this.$http({
        url: "/upload/checkSnippet",
        method: "post",
        data: { hash }
      })
    },
    // 小文件上传
    sendChunk(item) {
      if (!item) return
      let formdata = new FormData()
      formdata.append("file", item.chunk)
      formdata.append("index", item.index)
      formdata.append("hash", this.fileInfo.hash)
      // formdata.append("name", this.fileInfo.name)

      return this.$http({
        url: "/upload/snippet",
        method: "post",
        data: formdata,
        headers: { "Content-Type": "multipart/form-data" }
      })
    },
    // 文件上传方法
    sendFile(file) {
      let formdata = new FormData()
      formdata.append("file", file)

      this.$http({
        url: "/upload/file",
        method: "post",
        data: formdata,
        headers: { "Content-Type": "multipart/form-data" }
      }).then(({ data }) => {
        console.log(data, 'upload/file')
      })
    },
    // 请求合并
    chunkMerge() {
      this.$http({
        url: "/upload/merge",
        method: "post",
        data: this.fileInfo,
      }).then(res => {
        console.log(res.data)
      })
    }
  }
}
</script>

vue.config.js

module.exports = {
  configureWebpack: config => {
    config.module.rules.push(
      {
        test: /\.worker\.js$/,
        use: { loader: "worker-loader" }
      }
    )
  },
  parallel: false,
  chainWebpack: config => {
    config.output.globalObject('this')
  }
}

服务端（server）

index.js

const Koa = require('koa')
const router = require('koa-router')() // koa路由模块
const koaBody = require('koa-body') //解析文件上传的插件
const fs = require('fs') // nodeJs内置文件模块
const path = require('path') // nodeJs内置路径模块

const uploadPath = path.join(__dirname, 'public/uploads') // 定义文件上传目录

// 如果初始没有改文件目录，则自动创建
if (!fs.existsSync(uploadPath)) {
  fs.mkdirSync(uploadPath)
}

const app = new Koa() // 实例化

// 一些自定义的全局请求处理
app.use(async (ctx, next) => {
  console.log(`Process ${ctx.request.method} ${ctx.request.url}...`);

  if (ctx.request.method === 'OPTIONS') {
    ctx.status = 200
  }

  try {
    await next();
  } catch (err) {
    ctx.status = err.statusCode || err.status || 500
    ctx.body = {
      code: 500,
      msg: err.message
    }
  }
})

// 加载文件上传中间件
app.use(koaBody({
  multipart: true,
  formidable: {
    // keepExtensions: true, // 保持文件后缀
    uploadDir: uploadPath, // 初始指定文件存放地址，否则将会放入系统临时文件目录
    maxFileSize: 10000 * 1024 * 1024    // 设置上传文件大小最大限制，默认20M
  }
}))

// 文件上传处理
function uploadFn(ctx, destPath) {
  return new Promise((resolve, reject) => {
    const { name, path: _path } = ctx.request.files.file // 拿到上传的文件信息
    const filePath = destPath || path.join(uploadPath, name) // 重新组合文件名

    // 将临时文件重新设置文件名及地址
    fs.rename(_path, filePath, (err) => {
      if (err) {
        return reject(err)
      }
      resolve(filePath)
    })
  })
}

// 查询分片文件是否上传
router.post('/api/upload/checkSnippet', function snippet(ctx) {
  const { hash } = ctx.request.body

  // 切片上传目录
  const chunksPath = path.join(uploadPath, hash, '/')

  let chunksFiles = []

  if(fs.existsSync(chunksPath)) {
    // 切片文件
    chunksFiles = fs.readdirSync(chunksPath)
  }

  ctx.body = {
    code: 0,
    data: chunksFiles,
    msg: '查询成功'
  }
})

// 分片文件上传接口
router.post('/api/upload/snippet', async function snippet(ctx) {
  const { index, hash } = ctx.request.body

  // 切片上传目录
  const chunksPath = path.join(uploadPath, hash, '/')

  if(!fs.existsSync(chunksPath)) {
    fs.mkdirSync(chunksPath)
  }

  // 切片文件
  const chunksFileName = chunksPath + hash + '-' + index
  
  await uploadFn(ctx, chunksFileName).then(name => {
    ctx.body = {
      code: 0,
      msg: '切片上传完成',
      data: name
    }
  }).catch(err => {
    ctx.body = {
      code: -1,
      msg: '切片上传失败',
      data: err
    }
  })
})

// 文件上传接口
router.post('/api/upload/file', async function uploadFile(ctx) {
  await uploadFn(ctx).then((name) => {
    ctx.body = {
      code: 0,
      url: path.join('http://localhost:3000/uploads', name),
      msg: '文件上传成功'
    }
  }).catch(err => {
    ctx.body = {
      code: -1,
      msg: '文件上传失败'
    }
  })
})

// 删除文件夹及内部所有文件
function deleteFiles(dirpath) {
  if (fs.existsSync(dirpath)) {
    fs.readdir(dirpath, (err, files) => {
      if (err) throw err
      // 删除文件
      while(files.length) {
        fs.unlinkSync(dirpath + files.shift())
      }
      // 删除目录
      fs.rmdir(dirpath, () => {})
    })
  }
}
/**
 * 文件异步合并
 * @param {String} dirPath 分片文件夹
 * @param {String} filePath 目标文件
 * @param {String} hash 文件hash
 * @param {Number} total 分片文件总数
 * @returns {Promise}
 */
function mergeFile(dirPath, filePath, hash, total) {
  return new Promise((resolve, reject) => {
    fs.readdir(dirPath, (err, files) => {
      if (err) {
        return reject(err)
      }
      if(files.length !== total || !files.length) {
        return reject('上传失败，切片数量不符')
      }

      // 创建文件写入流
      const fileWriteStream = fs.createWriteStream(filePath)
      function merge(i) {
        return new Promise((res, rej) => {
          // 合并完成
          if (i === files.length) {
            fs.rmdir(dirPath, (err) => {
              console.log(err, 'rmdir')
            })
            return res()
          }
          const chunkpath = dirPath + hash + '-' + i
          fs.readFile(chunkpath, (err, data) => {
            if (err) return rej(err)

            // 将切片追加到存储文件
            fs.appendFile(filePath, data, () => {
              // 删除切片文件
              fs.unlink(chunkpath, () => {
                // 递归合并
                res(merge(i + 1))
              })
            })
          })

        })
      }
      merge(0).then(() => {
        // 默认情况下不需要手动关闭，但是在某些文件的合并并不会自动关闭可写流，比如压缩文件，所以这里在合并完成之后，统一关闭下
        resolve(fileWriteStream.close())
      })
    })
  })
}

/**
 * 文件合并接口
 * 1、判断是否有切片hash文件夹
 * 2、判断文件夹内的文件数量是否等于total
 * 4、然后合并切片
 * 5、删除切片文件信息
 */
router.post('/api/upload/merge', async function uploadFile(ctx) {
  const { total, hash, name } = ctx.request.body
  const dirPath = path.join(uploadPath, hash, '/')
  const filePath = path.join(uploadPath, name) // 合并文件

  // 已存在文件，则表示已上传成功
  if (fs.existsSync(filePath)) {
    // 删除所有的临时文件
    deleteFiles(dirPath)
    ctx.body = {
      code: 0,
      url: path.join('http://localhost:3000/uploads', name),
      msg: '文件上传成功'
    }
  // 如果没有切片hash文件夹则表明上传失败
  } else if (!fs.existsSync(dirPath)) {
    ctx.body = {
      code: -1,
      msg: '文件上传失败'
    }
  } else {
    // 开始合并
    await mergeFile(dirPath, filePath, hash, total).then(() => {
      ctx.body = {
        code: 0,
        url: path.join('http://localhost:3000/uploads', name),
        msg: '文件上传成功'
      }
    }).catch(err => {
      ctx.body = {
        code: -1,
        msg: err
      }
    })
  }
})

app.use(router.routes())

// 用端口3000启动服务
app.listen(3000)

Git地址

https://gitee.com/zlluGitHub/big-file-upload