update w/ iwestlin -7dd7998
This commit is contained in:
liaojack8
parent
8ff4fbc4fa
commit
ebeda6c41a
74
src/gd.js
74
src/gd.js
|
|
@ -12,8 +12,9 @@ const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY,
|
|||
const { db } = require('../db')
|
||||
const { make_table, make_tg_table, make_html, summary } = require('./summary')
|
||||
|
||||
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬)，停止複製'
|
||||
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬)，停止複製，請將未完成的資料夾移到另一個小組雲端硬碟中，再執行一遍複製指令即可繼斷點續傳'
|
||||
const FOLDER_TYPE = 'application/vnd.google-apps.folder'
|
||||
const sleep = ms => new Promise((resolve, reject) => setTimeout(resolve, ms))
|
||||
const { https_proxy } = process.env
|
||||
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
|
||||
|
||||
|
|
@ -387,7 +388,7 @@ async function get_info_by_id (fid, use_sa) {
|
|||
includeItemsFromAllDrives: true,
|
||||
supportsAllDrives: true,
|
||||
corpora: 'allDrives',
|
||||
fields: 'id,name'
|
||||
fields: 'id, parents'
|
||||
}
|
||||
url += '?' + params_to_query(params)
|
||||
const headers = await gen_headers(use_sa)
|
||||
|
|
@ -521,25 +522,64 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
|
|||
}
|
||||
|
||||
async function copy_files ({ files, mapping, service_account, root, task_id }) {
|
||||
if (!files.length) return
|
||||
console.log('\n開始複製文件，總數：', files.length)
|
||||
const limit = pLimit(PARALLEL_LIMIT)
|
||||
let count = 0
|
||||
const loop = setInterval(() => {
|
||||
const now = dayjs().format('HH:mm:ss')
|
||||
const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
|
||||
const message = `${now} | 已複製的檔案數 ${count} | 排隊中檔案數${files.length}`
|
||||
print_progress(message)
|
||||
}, 1000)
|
||||
return Promise.all(files.map(async file => {
|
||||
|
||||
let count = 0
|
||||
let concurrency = 0
|
||||
let err
|
||||
do {
|
||||
if (err) {
|
||||
clearInterval(loop)
|
||||
throw err
|
||||
}
|
||||
if (concurrency > PARALLEL_LIMIT) {
|
||||
await sleep(100)
|
||||
continue
|
||||
}
|
||||
const file = files.shift()
|
||||
if (!file) {
|
||||
await sleep(1000)
|
||||
continue
|
||||
}
|
||||
concurrency++
|
||||
const { id, parent } = file
|
||||
const target = mapping[parent] || root
|
||||
const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
|
||||
copy_file(id, target, service_account, null, task_id).then(new_file => {
|
||||
if (new_file) {
|
||||
count++
|
||||
db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
|
||||
}
|
||||
})).finally(() => clearInterval(loop))
|
||||
}).catch(e => {
|
||||
err = e
|
||||
}).finally(() => {
|
||||
concurrency--
|
||||
})
|
||||
} while (concurrency)
|
||||
clearInterval(loop)
|
||||
// const limit = pLimit(PARALLEL_LIMIT)
|
||||
// let count = 0
|
||||
// const loop = setInterval(() => {
|
||||
// const now = dayjs().format('HH:mm:ss')
|
||||
// const {activeCount, pendingCount} = limit
|
||||
// const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${activeCount}/排队中${pendingCount}`
|
||||
// print_progress(message)
|
||||
// }, 1000)
|
||||
// return Promise.all(files.map(async file => {
|
||||
// const { id, parent } = file
|
||||
// const target = mapping[parent] || root
|
||||
// const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
|
||||
// if (new_file) {
|
||||
// count++
|
||||
// db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
|
||||
// }
|
||||
// })).finally(() => clearInterval(loop))
|
||||
}
|
||||
|
||||
async function copy_file (id, parent, use_sa, limit, task_id) {
|
||||
let url = `https://www.googleapis.com/drive/v3/files/${id}/copy`
|
||||
let params = { supportsAllDrives: true }
|
||||
|
|
@ -679,6 +719,22 @@ async function confirm_dedupe ({ file_number, folder_number }) {
|
|||
return answer.value
|
||||
}
|
||||
|
||||
// 需要sa是源文件夹所在盘的manager
|
||||
async function mv_file ({ fid, new_parent, service_account }) {
|
||||
const file = await get_info_by_id(fid)
|
||||
if (!file) return
|
||||
const removeParents = file.parents[0]
|
||||
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
|
||||
const params = {
|
||||
removeParents,
|
||||
supportsAllDrives: true,
|
||||
addParents: new_parent
|
||||
}
|
||||
url += '?' + params_to_query(params)
|
||||
const headers = await gen_headers(service_account)
|
||||
return axins.patch(url, {}, { headers })
|
||||
}
|
||||
|
||||
// 将文件或文件夹移入回收站，需要 sa 为 content manager 权限及以上
|
||||
async function trash_file ({ fid, service_account }) {
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true`
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ const Router = require('@koa/router')
|
|||
|
||||
const { db } = require('../db')
|
||||
const { validate_fid, gen_count_body } = require('./gd')
|
||||
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg')
|
||||
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task } = require('./tg')
|
||||
|
||||
const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config')
|
||||
const { tg_whitelist } = AUTH
|
||||
|
|
@ -126,6 +126,15 @@ router.post('/api/gdurl/tgbot', async ctx => {
|
|||
let task_id = text.replace('/task', '').trim()
|
||||
if (task_id === 'all') {
|
||||
return send_all_tasks(chat_id)
|
||||
} else if (task_id === 'clear') {
|
||||
return clear_tasks(chat_id)
|
||||
} else if (task_id === '-h') {
|
||||
return send_task_help(chat_id)
|
||||
} else if (task_id.startsWith('rm')) {
|
||||
task_id = task_id.replace('rm', '')
|
||||
task_id = parseInt(task_id)
|
||||
if (!task_id) return send_task_help(chat_id)
|
||||
return rm_task({ task_id, chat_id })
|
||||
}
|
||||
task_id = parseInt(task_id)
|
||||
if (!task_id) {
|
||||
|
|
|
|||
42
src/tg.js
42
src/tg.js
|
|
@ -39,6 +39,11 @@ sourceID可以是共享網址本身，也可以是共享ID。如果命令最后
|
|||
=====================
|
||||
/task taskID(選填) | 返回對應任務的進度信息，若不填taskID則返回所有正在運行的任務進度
|
||||
若填 all 則返回所有任務列表(歷史紀錄)
|
||||
/task | 返回所有正在執行的正在執行的任務詳情
|
||||
/task 7 | 返回ID为 7 的任務詳情
|
||||
/task all | 返回所有任務紀錄列表
|
||||
/task clear | 清除所有狀態為finished的任務紀錄
|
||||
/task rm 7 | 刪除編號為 7 的任務紀錄
|
||||
=====================
|
||||
/bm [action] [alias] [target] | bookmark，添加常用目的資料夾ID
|
||||
會在輸入共享連結後返回的「文件統計」「開始複製」這兩個按鈕的下方出現，方便複製到常用位置。
|
||||
|
|
@ -61,6 +66,32 @@ function send_bm_help (chat_id) {
|
|||
return sm({ chat_id, text, parse_mode: 'HTML' })
|
||||
}
|
||||
|
||||
function send_task_help (chat_id) {
|
||||
const text = `<pre>/task [action/id] [id] | 查詢或管理任務進度
|
||||
範例：
|
||||
/task | 返回所有正在執行的正在執行的任務詳情
|
||||
/task 7 | 返回ID为 7 的任務詳情
|
||||
/task all | 返回所有任務紀錄列表
|
||||
/task clear | 清除所有狀態為finished的任務紀錄
|
||||
/task rm 7 | 刪除編號為 7 的任務紀錄
|
||||
</pre>`
|
||||
return sm({ chat_id, text, parse_mode: 'HTML' })
|
||||
}
|
||||
|
||||
function clear_tasks (chat_id) {
|
||||
const finished_tasks = db.prepare('select id from task where status=?').all('finished')
|
||||
finished_tasks.forEach(task => rm_task({ task_id: task.id }))
|
||||
sm({ chat_id, text: '已清除所有狀態為finished的任務紀錄' })
|
||||
}
|
||||
|
||||
function rm_task ({ task_id, chat_id }) {
|
||||
const exist = db.prepare('select id from task where id=?').get(task_id)
|
||||
if (!exist) return sm({ chat_id, text: `不存在编号为 ${task_id} 的任务记录` })
|
||||
db.prepare('delete from task where id=?').run(task_id)
|
||||
db.prepare('delete from copied where taskid=?').run(task_id)
|
||||
if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 紀錄` })
|
||||
}
|
||||
|
||||
function send_all_bookmarks (chat_id) {
|
||||
let records = db.prepare('select alias, target from bookmark').all()
|
||||
if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
|
||||
|
|
@ -91,6 +122,11 @@ function get_target_by_alias (alias) {
|
|||
return record && record.target
|
||||
}
|
||||
|
||||
function get_alias_by_target (target) {
|
||||
const record = db.prepare('select alias from bookmark where target=?').get(target)
|
||||
return record && record.alias
|
||||
}
|
||||
|
||||
function send_choice ({ fid, chat_id }) {
|
||||
if(BUTTON_LEVEL == 1){
|
||||
return sm({
|
||||
|
|
@ -212,8 +248,8 @@ async function send_task_info ({ task_id, chat_id }) {
|
|||
} catch (e) {
|
||||
console.log('fail to send message to tg', e.message)
|
||||
}
|
||||
// get_task_info 在task目录数超大时比较吃cpu，如果超1万就不每10秒更新了，以后如果把mapping 也另存一张表可以取消此限制
|
||||
if (!message_id || status !== 'copying' || folder_count > 10000) return
|
||||
// get_task_info 在task目录数超大时比较吃cpu，以后最好把mapping也另存一张表
|
||||
if (!message_id || status !== 'copying') return
|
||||
const loop = setInterval(async () => {
|
||||
const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
|
||||
const { text, status } = await get_task_info(task_id)
|
||||
|
|
@ -347,4 +383,4 @@ function extract_from_text (text) {
|
|||
return m && extract_fid(m[0])
|
||||
}
|
||||
|
||||
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark }
|
||||
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task }
|
||||
|
|
|
|||
Loading…
Reference in New Issue