update w/ iwestlin -7dd7998

src/gd.js

@@ -12,8 +12,9 @@ const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY,
 const { db } = require('../db')
 const { make_table, make_tg_table, make_html, summary } = require('./summary')
 
-const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬)，停止複製'
+const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬)，停止複製，請將未完成的資料夾移到另一個小組雲端硬碟中，再執行一遍複製指令即可繼斷點續傳'
 const FOLDER_TYPE = 'application/vnd.google-apps.folder'
+const sleep = ms => new Promise((resolve, reject) => setTimeout(resolve, ms))
 const { https_proxy } = process.env
 const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
 
@@ -387,7 +388,7 @@ async function get_info_by_id (fid, use_sa) {
     includeItemsFromAllDrives: true,
     supportsAllDrives: true,
     corpora: 'allDrives',
-    fields: 'id,name'
+    fields: 'id, parents'
   }
   url += '?' + params_to_query(params)
   const headers = await gen_headers(use_sa)
@@ -521,25 +522,64 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
 }
 
 async function copy_files ({ files, mapping, service_account, root, task_id }) {
+  if (!files.length) return
   console.log('\n開始複製文件，總數：', files.length)
-  const limit = pLimit(PARALLEL_LIMIT)
-  let count = 0
   const loop = setInterval(() => {
     const now = dayjs().format('HH:mm:ss')
-    const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
+    const message = `${now} | 已複製的檔案數 ${count} | 排隊中檔案數${files.length}`
     print_progress(message)
   }, 1000)
-  return Promise.all(files.map(async file => {
+
+  let count = 0
+  let concurrency = 0
+  let err
+  do {
+    if (err) {
+      clearInterval(loop)
+      throw err
+    }
+    if (concurrency > PARALLEL_LIMIT) {
+      await sleep(100)
+      continue
+    }
+    const file = files.shift()
+    if (!file) {
+      await sleep(1000)
+      continue
+    }
+    concurrency++
     const { id, parent } = file
     const target = mapping[parent] || root
-    const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
+    copy_file(id, target, service_account, null, task_id).then(new_file => {
       if (new_file) {
         count++
         db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
       }
-  })).finally(() => clearInterval(loop))
+    }).catch(e => {
+      err = e
+    }).finally(() => {
+      concurrency--
+    })
+  } while (concurrency)
+  clearInterval(loop)
+  // const limit = pLimit(PARALLEL_LIMIT)
+  // let count = 0
+  // const loop = setInterval(() => {
+  //   const now = dayjs().format('HH:mm:ss')
+  //   const {activeCount, pendingCount} = limit
+  //   const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${activeCount}/排队中${pendingCount}`
+  //   print_progress(message)
+  // }, 1000)
+  // return Promise.all(files.map(async file => {
+  //   const { id, parent } = file
+  //   const target = mapping[parent] || root
+  //   const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
+  //   if (new_file) {
+  //     count++
+  //     db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
+  //   }
+  // })).finally(() => clearInterval(loop))
 }
-
 async function copy_file (id, parent, use_sa, limit, task_id) {
   let url = `https://www.googleapis.com/drive/v3/files/${id}/copy`
   let params = { supportsAllDrives: true }
@@ -679,6 +719,22 @@ async function confirm_dedupe ({ file_number, folder_number }) {
   return answer.value
 }
 
+// 需要sa是源文件夹所在盘的manager
+async function mv_file ({ fid, new_parent, service_account }) {
+  const file = await get_info_by_id(fid)
+  if (!file) return
+  const removeParents = file.parents[0]
+  let url = `https://www.googleapis.com/drive/v3/files/${fid}`
+  const params = {
+    removeParents,
+    supportsAllDrives: true,
+    addParents: new_parent
+  }
+  url += '?' + params_to_query(params)
+  const headers = await gen_headers(service_account)
+  return axins.patch(url, {}, { headers })
+}
+
 // 将文件或文件夹移入回收站，需要 sa 为 content manager 权限及以上
 async function trash_file ({ fid, service_account }) {
   const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true`

src/router.js

@@ -2,7 +2,7 @@ const Router = require('@koa/router')
 
 const { db } = require('../db')
 const { validate_fid, gen_count_body } = require('./gd')
-const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg')
+const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task } = require('./tg')
 
 const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config')
 const { tg_whitelist } = AUTH
@@ -126,6 +126,15 @@ router.post('/api/gdurl/tgbot', async ctx => {
     let task_id = text.replace('/task', '').trim()
     if (task_id === 'all') {
       return send_all_tasks(chat_id)
+    } else if (task_id === 'clear') {
+      return clear_tasks(chat_id)
+    } else if (task_id === '-h') {
+      return send_task_help(chat_id)
+    } else if (task_id.startsWith('rm')) {
+      task_id = task_id.replace('rm', '')
+      task_id = parseInt(task_id)
+      if (!task_id) return send_task_help(chat_id)
+      return rm_task({ task_id, chat_id })
     }
     task_id = parseInt(task_id)
     if (!task_id) {

src/tg.js

@@ -39,6 +39,11 @@ sourceID可以是共享網址本身，也可以是共享ID。如果命令最后
 =====================
 /task taskID(選填) | 返回對應任務的進度信息，若不填taskID則返回所有正在運行的任務進度
 若填 all 則返回所有任務列表(歷史紀錄)
+/task | 返回所有正在執行的正在執行的任務詳情
+/task 7 | 返回ID为 7 的任務詳情
+/task all | 返回所有任務紀錄列表
+/task clear | 清除所有狀態為finished的任務紀錄
+/task rm 7 | 刪除編號為 7 的任務紀錄
 =====================
 /bm [action] [alias] [target] | bookmark，添加常用目的資料夾ID
 會在輸入共享連結後返回的「文件統計」「開始複製」這兩個按鈕的下方出現，方便複製到常用位置。
@@ -61,6 +66,32 @@ function send_bm_help (chat_id) {
   return sm({ chat_id, text, parse_mode: 'HTML' })
 }
 
+function send_task_help (chat_id) {
+  const text = `<pre>/task [action/id] [id] | 查詢或管理任務進度
+範例：
+/task | 返回所有正在執行的正在執行的任務詳情
+/task 7 | 返回ID为 7 的任務詳情
+/task all | 返回所有任務紀錄列表
+/task clear | 清除所有狀態為finished的任務紀錄
+/task rm 7 | 刪除編號為 7 的任務紀錄
+</pre>`
+  return sm({ chat_id, text, parse_mode: 'HTML' })
+}
+
+function clear_tasks (chat_id) {
+  const finished_tasks = db.prepare('select id from task where status=?').all('finished')
+  finished_tasks.forEach(task => rm_task({ task_id: task.id }))
+  sm({ chat_id, text: '已清除所有狀態為finished的任務紀錄' })
+}
+
+function rm_task ({ task_id, chat_id }) {
+  const exist = db.prepare('select id from task where id=?').get(task_id)
+  if (!exist) return sm({ chat_id, text: `不存在编号为 ${task_id} 的任务记录` })
+  db.prepare('delete from task where id=?').run(task_id)
+  db.prepare('delete from copied where taskid=?').run(task_id)
+  if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 紀錄` })
+}
+
 function send_all_bookmarks (chat_id) {
   let records = db.prepare('select alias, target from bookmark').all()
   if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
@@ -91,6 +122,11 @@ function get_target_by_alias (alias) {
   return record && record.target
 }
 
+function get_alias_by_target (target) {
+  const record = db.prepare('select alias from bookmark where target=?').get(target)
+  return record && record.alias
+}
+
 function send_choice ({ fid, chat_id }) {
   if(BUTTON_LEVEL == 1){
   	return sm({
@@ -212,8 +248,8 @@ async function send_task_info ({ task_id, chat_id }) {
   } catch (e) {
     console.log('fail to send message to tg', e.message)
   }
-  // get_task_info 在task目录数超大时比较吃cpu，如果超1万就不每10秒更新了，以后如果把mapping 也另存一张表可以取消此限制
+  // get_task_info 在task目录数超大时比较吃cpu，以后最好把mapping也另存一张表
-  if (!message_id || status !== 'copying' || folder_count > 10000) return
+  if (!message_id || status !== 'copying') return
   const loop = setInterval(async () => {
     const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
     const { text, status } = await get_task_info(task_id)
@@ -347,4 +383,4 @@ function extract_from_text (text) {
   return m && extract_fid(m[0])
 }
 
-module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark }
+module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task }