diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..317e638 Binary files /dev/null and b/.DS_Store differ diff --git a/src/gd.js b/src/gd.js index a7f16b3..670b304 100644 --- a/src/gd.js +++ b/src/gd.js @@ -12,8 +12,9 @@ const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, const { db } = require('../db') const { make_table, make_tg_table, make_html, summary } = require('./summary') -const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製' +const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製,請將未完成的資料夾移到另一個小組雲端硬碟中,再執行一遍複製指令即可繼斷點續傳' const FOLDER_TYPE = 'application/vnd.google-apps.folder' +const sleep = ms => new Promise((resolve, reject) => setTimeout(resolve, ms)) const { https_proxy } = process.env const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {}) @@ -387,7 +388,7 @@ async function get_info_by_id (fid, use_sa) { includeItemsFromAllDrives: true, supportsAllDrives: true, corpora: 'allDrives', - fields: 'id,name' + fields: 'id, parents' } url += '?' + params_to_query(params) const headers = await gen_headers(use_sa) @@ -521,25 +522,64 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t } async function copy_files ({ files, mapping, service_account, root, task_id }) { + if (!files.length) return console.log('\n開始複製文件,總數:', files.length) - const limit = pLimit(PARALLEL_LIMIT) - let count = 0 const loop = setInterval(() => { const now = dayjs().format('HH:mm:ss') - const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}` + const message = `${now} | 已複製的檔案數 ${count} | 排隊中檔案數${files.length}` print_progress(message) }, 1000) - return Promise.all(files.map(async file => { + + let count = 0 + let concurrency = 0 + let err + do { + if (err) { + clearInterval(loop) + throw err + } + if (concurrency > PARALLEL_LIMIT) { + await sleep(100) + continue + } + const file = files.shift() + if (!file) { + await sleep(1000) + continue + } + concurrency++ const { id, parent } = file const target = mapping[parent] || root - const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id)) - if (new_file) { - count++ - db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id) - } - })).finally(() => clearInterval(loop)) + copy_file(id, target, service_account, null, task_id).then(new_file => { + if (new_file) { + count++ + db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id) + } + }).catch(e => { + err = e + }).finally(() => { + concurrency-- + }) + } while (concurrency) + clearInterval(loop) + // const limit = pLimit(PARALLEL_LIMIT) + // let count = 0 + // const loop = setInterval(() => { + // const now = dayjs().format('HH:mm:ss') + // const {activeCount, pendingCount} = limit + // const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${activeCount}/排队中${pendingCount}` + // print_progress(message) + // }, 1000) + // return Promise.all(files.map(async file => { + // const { id, parent } = file + // const target = mapping[parent] || root + // const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id)) + // if (new_file) { + // count++ + // db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id) + // } + // })).finally(() => clearInterval(loop)) } - async function copy_file (id, parent, use_sa, limit, task_id) { let url = `https://www.googleapis.com/drive/v3/files/${id}/copy` let params = { supportsAllDrives: true } @@ -679,6 +719,22 @@ async function confirm_dedupe ({ file_number, folder_number }) { return answer.value } +// 需要sa是源文件夹所在盘的manager +async function mv_file ({ fid, new_parent, service_account }) { + const file = await get_info_by_id(fid) + if (!file) return + const removeParents = file.parents[0] + let url = `https://www.googleapis.com/drive/v3/files/${fid}` + const params = { + removeParents, + supportsAllDrives: true, + addParents: new_parent + } + url += '?' + params_to_query(params) + const headers = await gen_headers(service_account) + return axins.patch(url, {}, { headers }) +} + // 将文件或文件夹移入回收站,需要 sa 为 content manager 权限及以上 async function trash_file ({ fid, service_account }) { const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true` diff --git a/src/router.js b/src/router.js index 1cf3572..195bab0 100644 --- a/src/router.js +++ b/src/router.js @@ -2,7 +2,7 @@ const Router = require('@koa/router') const { db } = require('../db') const { validate_fid, gen_count_body } = require('./gd') -const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg') +const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task } = require('./tg') const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config') const { tg_whitelist } = AUTH @@ -126,6 +126,15 @@ router.post('/api/gdurl/tgbot', async ctx => { let task_id = text.replace('/task', '').trim() if (task_id === 'all') { return send_all_tasks(chat_id) + } else if (task_id === 'clear') { + return clear_tasks(chat_id) + } else if (task_id === '-h') { + return send_task_help(chat_id) + } else if (task_id.startsWith('rm')) { + task_id = task_id.replace('rm', '') + task_id = parseInt(task_id) + if (!task_id) return send_task_help(chat_id) + return rm_task({ task_id, chat_id }) } task_id = parseInt(task_id) if (!task_id) { diff --git a/src/tg.js b/src/tg.js index f95de84..5b1ee63 100644 --- a/src/tg.js +++ b/src/tg.js @@ -39,6 +39,11 @@ sourceID可以是共享網址本身,也可以是共享ID。如果命令最后 ===================== /task taskID(選填) | 返回對應任務的進度信息,若不填taskID則返回所有正在運行的任務進度 若填 all 則返回所有任務列表(歷史紀錄) +/task | 返回所有正在執行的正在執行的任務詳情 +/task 7 | 返回ID为 7 的任務詳情 +/task all | 返回所有任務紀錄列表 +/task clear | 清除所有狀態為finished的任務紀錄 +/task rm 7 | 刪除編號為 7 的任務紀錄 ===================== /bm [action] [alias] [target] | bookmark,添加常用目的資料夾ID 會在輸入共享連結後返回的「文件統計」「開始複製」這兩個按鈕的下方出現,方便複製到常用位置。 @@ -61,6 +66,32 @@ function send_bm_help (chat_id) { return sm({ chat_id, text, parse_mode: 'HTML' }) } +function send_task_help (chat_id) { + const text = `
/task [action/id] [id] | 查詢或管理任務進度
+範例:
+/task | 返回所有正在執行的正在執行的任務詳情
+/task 7 | 返回ID为 7 的任務詳情
+/task all | 返回所有任務紀錄列表
+/task clear | 清除所有狀態為finished的任務紀錄
+/task rm 7 | 刪除編號為 7 的任務紀錄
+
` + return sm({ chat_id, text, parse_mode: 'HTML' }) +} + +function clear_tasks (chat_id) { + const finished_tasks = db.prepare('select id from task where status=?').all('finished') + finished_tasks.forEach(task => rm_task({ task_id: task.id })) + sm({ chat_id, text: '已清除所有狀態為finished的任務紀錄' }) +} + +function rm_task ({ task_id, chat_id }) { + const exist = db.prepare('select id from task where id=?').get(task_id) + if (!exist) return sm({ chat_id, text: `不存在编号为 ${task_id} 的任务记录` }) + db.prepare('delete from task where id=?').run(task_id) + db.prepare('delete from copied where taskid=?').run(task_id) + if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 紀錄` }) +} + function send_all_bookmarks (chat_id) { let records = db.prepare('select alias, target from bookmark').all() if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' }) @@ -91,6 +122,11 @@ function get_target_by_alias (alias) { return record && record.target } +function get_alias_by_target (target) { + const record = db.prepare('select alias from bookmark where target=?').get(target) + return record && record.alias +} + function send_choice ({ fid, chat_id }) { if(BUTTON_LEVEL == 1){ return sm({ @@ -212,8 +248,8 @@ async function send_task_info ({ task_id, chat_id }) { } catch (e) { console.log('fail to send message to tg', e.message) } - // get_task_info 在task目录数超大时比较吃cpu,如果超1万就不每10秒更新了,以后如果把mapping 也另存一张表可以取消此限制 - if (!message_id || status !== 'copying' || folder_count > 10000) return + // get_task_info 在task目录数超大时比较吃cpu,以后最好把mapping也另存一张表 + if (!message_id || status !== 'copying') return const loop = setInterval(async () => { const url = `https://api.telegram.org/bot${tg_token}/editMessageText` const { text, status } = await get_task_info(task_id) @@ -347,4 +383,4 @@ function extract_from_text (text) { return m && extract_fid(m[0]) } -module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark } +module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task }