update bookmark func

This commit is contained in:
liaojack8 2020-07-06 18:48:19 +08:00
parent 31ca1ead39
commit 54965cf5cb
6 changed files with 210 additions and 133 deletions

View File

@ -33,4 +33,13 @@ CREATE TABLE "copied" (
"fileid" TEXT "fileid" TEXT
); );
CREATE INDEX "copied_taskid" ON "copied" ("taskid"); CREATE INDEX "copied_taskid" ON "copied" ("taskid");
CREATE TABLE "bookmark" (
"alias" TEXT,
"target" TEXT
);
CREATE UNIQUE INDEX "bookmark_alias" ON "bookmark" (
"alias"
);

22
db.js
View File

@ -2,12 +2,11 @@ const path = require('path')
const db_location = path.join(__dirname, 'gdurl.sqlite') const db_location = path.join(__dirname, 'gdurl.sqlite')
const db = require('better-sqlite3')(db_location) const db = require('better-sqlite3')(db_location)
module.exports = { db } db.pragma('journal_mode = WAL')
create_table_copied() create_table_copied()
function create_table_copied () { function create_table_copied () {
const [exists] = db.prepare('PRAGMA table_info(copied)').all() const [exists] = db.prepare('PRAGMA table_info(copied)').all()
// console.log('exists', exists)
if (exists) return if (exists) return
const create_table = `CREATE TABLE "copied" ( const create_table = `CREATE TABLE "copied" (
"taskid" INTEGER, "taskid" INTEGER,
@ -16,4 +15,21 @@ function create_table_copied () {
db.prepare(create_table).run() db.prepare(create_table).run()
const create_index = `CREATE INDEX "copied_taskid" ON "copied" ("taskid");` const create_index = `CREATE INDEX "copied_taskid" ON "copied" ("taskid");`
db.prepare(create_index).run() db.prepare(create_index).run()
} }
create_table_bookmark()
function create_table_bookmark () {
const [exists] = db.prepare('PRAGMA table_info(bookmark)').all()
if (exists) return
const create_table = `CREATE TABLE "bookmark" (
"alias" TEXT,
"target" TEXT
);`
db.prepare(create_table).run()
const create_index = `CREATE UNIQUE INDEX "bookmark_alias" ON "bookmark" (
"alias"
);`
db.prepare(create_index).run()
}
module.exports = { db }

131
src/gd.js
View File

@ -97,7 +97,7 @@ async function count ({ fid, update, sort, type, output, not_teamdrive, service_
if (!update) { if (!update) {
const info = get_all_by_fid(fid) const info = get_all_by_fid(fid)
if (info) { if (info) {
console.log('找到本地缓存数据,缓存时间', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss')) console.log('找到本地快取資料,快取時間', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss'))
const out_str = get_out_str({ info, type, sort }) const out_str = get_out_str({ info, type, sort })
if (output) return fs.writeFileSync(output, out_str) if (output) return fs.writeFileSync(output, out_str)
return console.log(out_str) return console.log(out_str)
@ -165,7 +165,7 @@ async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
const loop = setInterval(() => { const loop = setInterval(() => {
const now = dayjs().format('HH:mm:ss') const now = dayjs().format('HH:mm:ss')
const message = `${now} | 已获取对象 ${result.length} | 网络请求 进行中${limit.activeCount}/排队${limit.pendingCount}` const message = `${now} | 已獲取對象 ${result.length} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
print_progress(message) print_progress(message)
}, 1000) }, 1000)
@ -196,8 +196,8 @@ async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
} catch (e) { } catch (e) {
console.error(e) console.error(e)
} }
console.log('\n信息获取完毕') console.log('\n資訊獲取完畢')
not_finished.length ? console.log('未读取完毕的目录ID', JSON.stringify(not_finished)) : console.log('所有目录读取完毕') not_finished.length ? console.log('未讀取完畢的目錄ID', JSON.stringify(not_finished)) : console.log('所有目錄讀取完畢')
clearInterval(loop) clearInterval(loop)
const smy = summary(result) const smy = summary(result)
db.prepare('UPDATE gd SET summary=?, mtime=? WHERE fid=?').run(JSON.stringify(smy), Date.now(), fid) db.prepare('UPDATE gd SET summary=?, mtime=? WHERE fid=?').run(JSON.stringify(smy), Date.now(), fid)
@ -228,7 +228,8 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
params.orderBy = 'folder,name desc' params.orderBy = 'folder,name desc'
params.fields = 'nextPageToken, files(id, name, mimeType, size, md5Checksum)' params.fields = 'nextPageToken, files(id, name, mimeType, size, md5Checksum)'
params.pageSize = Math.min(PAGE_SIZE, 1000) params.pageSize = Math.min(PAGE_SIZE, 1000)
const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa // const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa
const use_sa = (fid !== 'root') && service_account
const headers = await gen_headers(use_sa) const headers = await gen_headers(use_sa)
do { do {
if (pageToken) params.pageToken = pageToken if (pageToken) params.pageToken = pageToken
@ -247,7 +248,7 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
} }
} }
if (!data) { if (!data) {
console.error('读取目录未完成(部分读取), 参数:', params) console.error('讀取目錄未完成(部分讀取), 參數:', params)
files.not_finished = true files.not_finished = true
return files return files
} }
@ -300,26 +301,17 @@ async function get_sa_token () {
if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch() if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch()
} }
} }
throw new Error('没有可用的SA帐号') throw new Error('沒有可用的SA帳號')
} }
function real_get_sa_token (el) { async function real_get_sa_token (el) {
const { value, expires, gtoken } = el const { value, expires, gtoken } = el
// 把gtoken传递出去的原因是当某账号流量用尽时可以依此过滤 // 把gtoken传递出去的原因是当某账号流量用尽时可以依此过滤
if (Date.now() < expires) return { access_token: value, gtoken } if (Date.now() < expires) return { access_token: value, gtoken }
return new Promise((resolve, reject) => { const { access_token, expires_in } = await gtoken.getToken({ forceRefresh: true })
gtoken.getToken((err, tokens) => { el.value = access_token
if (err) { el.expires = Date.now() + 1000 * (expires_in - 60 * 5) // 提前5分钟判定为过期
reject(err) return { access_token, gtoken }
} else {
// console.log('got sa token', tokens)
const { access_token, expires_in } = tokens
el.value = access_token
el.expires = Date.now() + 1000 * (expires_in - 60 * 5) // 提前5分钟判定为过期
resolve({ access_token, gtoken })
}
})
})
} }
function get_random_element (arr) { function get_random_element (arr) {
@ -361,10 +353,10 @@ async function create_folder (name, parent, use_sa, limit) {
if (limit) limit.clearQueue() if (limit) limit.clearQueue()
throw new Error(FILE_EXCEED_MSG) throw new Error(FILE_EXCEED_MSG)
} }
console.log('创建目录重试中:', name, '重试次数', retry) console.log('創建目錄重試中:', name, '重試次數', retry)
} }
} }
throw new Error(err_message + ' 目名:' + name) throw new Error(err_message + ' 目名:' + name)
} }
async function get_name_by_id (fid) { async function get_name_by_id (fid) {
@ -394,10 +386,10 @@ async function user_choose () {
const answer = await prompts({ const answer = await prompts({
type: 'select', type: 'select',
name: 'value', name: 'value',
message: '检测到上次的复制记录,是否继续', message: '檢測到上次的複製紀錄,是否繼續',
choices: [ choices: [
{ title: 'Continue', description: '从上次中断的地方继续', value: 'continue' }, { title: 'Continue', description: '從上次中斷的地方繼續', value: 'continue' },
{ title: 'Restart', description: '无视已存在的记录,重新复制', value: 'restart' }, { title: 'Restart', description: '無視已存在的紀錄,重新複製', value: 'restart' },
{ title: 'Exit', description: '直接退出', value: 'exit' } { title: 'Exit', description: '直接退出', value: 'exit' }
], ],
initial: 0 initial: 0
@ -407,15 +399,15 @@ async function user_choose () {
async function copy ({ source, target, name, min_size, update, not_teamdrive, service_account, dncnr, is_server }) { async function copy ({ source, target, name, min_size, update, not_teamdrive, service_account, dncnr, is_server }) {
target = target || DEFAULT_TARGET target = target || DEFAULT_TARGET
if (!target) throw new Error('目标位置不能为空') if (!target) throw new Error('目標位置不能為空')
const record = db.prepare('select id, status from task where source=? and target=?').get(source, target) const record = db.prepare('select id, status from task where source=? and target=?').get(source, target)
if (record && record.status === 'copying') return console.log('已有相同源和目的地的任务正在运行,强制退出') if (record && record.status === 'copying') return console.log('已有相同來源和目的地的任務正在進行,強制退出')
try { try {
return await real_copy({ source, target, name, min_size, update, dncnr, not_teamdrive, service_account, is_server }) return await real_copy({ source, target, name, min_size, update, dncnr, not_teamdrive, service_account, is_server })
} catch (err) { } catch (err) {
console.error('复制文件夹出错', err) console.error('複製資料夾出錯', err)
const record = db.prepare('select id, status from task where source=? and target=?').get(source, target) const record = db.prepare('select id, status from task where source=? and target=?').get(source, target)
if (record) db.prepare('update task set status=? where id=?').run('error', record.id) if (record) db.prepare('update task set status=? where id=?').run('error', record.id)
} }
@ -435,14 +427,14 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
const record = db.prepare('select * from task where source=? and target=?').get(source, target) const record = db.prepare('select * from task where source=? and target=?').get(source, target)
if (record) { if (record) {
const copied = db.prepare('select fileid from copied where taskid=?').all(record.id).map(v => v.fileid)
const choice = is_server ? 'continue' : await user_choose() const choice = is_server ? 'continue' : await user_choose()
if (choice === 'exit') { if (choice === 'exit') {
return console.log('退出程序') return console.log('退出程序')
} else if (choice === 'continue') { } else if (choice === 'continue') {
let { copied, mapping } = record let { mapping } = record
const copied_ids = {}
const old_mapping = {} const old_mapping = {}
copied = copied.trim().split('\n') const copied_ids = {}
copied.forEach(id => copied_ids[id] = true) copied.forEach(id => copied_ids[id] = true)
mapping = mapping.trim().split('\n').map(line => line.split(' ')) mapping = mapping.trim().split('\n').map(line => line.split(' '))
const root = mapping[0][1] const root = mapping[0][1]
@ -451,9 +443,9 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account }) const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account })
let files = arr.filter(v => v.mimeType !== FOLDER_TYPE).filter(v => !copied_ids[v.id]) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE).filter(v => !copied_ids[v.id])
if (min_size) files = files.filter(v => v.size >= min_size) if (min_size) files = files.filter(v => v.size >= min_size)
const folders = arr.filter(v => v.mimeType === FOLDER_TYPE).filter(v => !old_mapping[v.id]) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE)
console.log('待复制的目录数', folders.length) console.log('待複製的目錄數', folders.length)
console.log('待复制的文件数', files.length) console.log('待複製的檔案數', files.length)
const all_mapping = await create_folders({ const all_mapping = await create_folders({
old_mapping, old_mapping,
source, source,
@ -468,14 +460,16 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
} else if (choice === 'restart') { } else if (choice === 'restart') {
const new_root = await get_new_root() const new_root = await get_new_root()
const root_mapping = source + ' ' + new_root.id + '\n' const root_mapping = source + ' ' + new_root.id + '\n'
db.prepare('update task set status=?, copied=?, mapping=? where id=?') db.prepare('update task set status=?, mapping=? where id=?').run('copying', root_mapping, record.id)
.run('copying', '', root_mapping, record.id) db.prepare('delete from copied where taskid=?').run(record.id)
const arr = await walk_and_save({ fid: source, update: true, not_teamdrive, service_account }) // const arr = await walk_and_save({ fid: source, update: true, not_teamdrive, service_account })
const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account })
let files = arr.filter(v => v.mimeType !== FOLDER_TYPE) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE)
if (min_size) files = files.filter(v => v.size >= min_size) if (min_size) files = files.filter(v => v.size >= min_size)
const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE)
console.log('待复制的目录数', folders.length) console.log('待複製的目錄數', folders.length)
console.log('待复制的文件数', files.length) console.log('待複製的檔案數', files.length)
const mapping = await create_folders({ const mapping = await create_folders({
source, source,
folders, folders,
@ -498,8 +492,8 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
let files = arr.filter(v => v.mimeType !== FOLDER_TYPE) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE)
if (min_size) files = files.filter(v => v.size >= min_size) if (min_size) files = files.filter(v => v.size >= min_size)
const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE)
console.log('待复制的目录数', folders.length) console.log('待複製的目錄數', folders.length)
console.log('待复制的文件数', files.length) console.log('待複製的檔案數', files.length)
const mapping = await create_folders({ const mapping = await create_folders({
source, source,
folders, folders,
@ -514,26 +508,26 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
} }
async function copy_files ({ files, mapping, service_account, root, task_id }) { async function copy_files ({ files, mapping, service_account, root, task_id }) {
console.log('\n开始复制文件,总数', files.length) console.log('\n開始複製文件,總數', files.length)
const limit = pLimit(PARALLEL_LIMIT) const limit = pLimit(PARALLEL_LIMIT)
let count = 0 let count = 0
const loop = setInterval(() => { const loop = setInterval(() => {
const now = dayjs().format('HH:mm:ss') const now = dayjs().format('HH:mm:ss')
const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${limit.activeCount}/排队${limit.pendingCount}` const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
print_progress(message) print_progress(message)
}, 1000) }, 1000)
return Promise.all(files.map(async file => { return Promise.all(files.map(async file => {
const { id, parent } = file const { id, parent } = file
const target = mapping[parent] || root const target = mapping[parent] || root
const new_file = await limit(() => copy_file(id, target, service_account, limit)) const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
if (new_file) { if (new_file) {
count++ count++
db.prepare('update task set status=?, copied = copied || ? where id=?').run('copying', id + '\n', task_id) db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
} }
})).finally(() => clearInterval(loop)) })).finally(() => clearInterval(loop))
} }
async function copy_file (id, parent, use_sa, limit) { async function copy_file (id, parent, use_sa, limit, task_id) {
let url = `https://www.googleapis.com/drive/v3/files/${id}/copy` let url = `https://www.googleapis.com/drive/v3/files/${id}/copy`
let params = { supportsAllDrives: true } let params = { supportsAllDrives: true }
url += '?' + params_to_query(params) url += '?' + params_to_query(params)
@ -558,21 +552,22 @@ async function copy_file (id, parent, use_sa, limit) {
const message = data && data.error && data.error.message const message = data && data.error && data.error.message
if (message && message.toLowerCase().includes('file limit')) { if (message && message.toLowerCase().includes('file limit')) {
if (limit) limit.clearQueue() if (limit) limit.clearQueue()
throw new Error('您的团队盘文件数已超限,停止复制') if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
throw new Error('您的小組雲端硬碟文件數已超限,停止複製')
} }
if (message && message.toLowerCase().includes('rate limit')) { if (use_sa && message && message.toLowerCase().includes('rate limit')) {
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken) SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken)
if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch() if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch()
console.log('此帐号触发使用限额剩余可用service account帐号数量:', SA_TOKENS.length) console.log('此帳號觸發使用限額剩餘可用service account帳號數量:', SA_TOKENS.length)
} }
} }
} }
if (use_sa && !SA_TOKENS.length) { if (use_sa && !SA_TOKENS.length) {
if (limit) limit.clearQueue() if (limit) limit.clearQueue()
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id) if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
throw new Error('所有SA帐号流量已用完') throw new Error('所有SA帳號流量已用完')
} else { } else {
console.warn('复制文件失败,文件id: ' + id) console.warn('複製檔案失敗,檔案id: ' + id)
} }
} }
@ -582,19 +577,21 @@ async function create_folders ({ source, old_mapping, folders, root, task_id, se
mapping[source] = root mapping[source] = root
if (!folders.length) return mapping if (!folders.length) return mapping
console.log('开始复制文件夹,总数:', folders.length) const missed_folders = folders.filter(v => !mapping[v.id])
console.log('開始複製資料夾,總數:', missed_folders.length)
const limit = pLimit(PARALLEL_LIMIT) const limit = pLimit(PARALLEL_LIMIT)
let count = 0 let count = 0
let same_levels = folders.filter(v => v.parent === folders[0].parent) let same_levels = folders.filter(v => v.parent === folders[0].parent)
const loop = setInterval(() => { const loop = setInterval(() => {
const now = dayjs().format('HH:mm:ss') const now = dayjs().format('HH:mm:ss')
const message = `${now} | 已创建目录数 ${count} | 网络请求 进行中${limit.activeCount}/排队${limit.pendingCount}` const message = `${now} | 已創建目錄 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
print_progress(message) print_progress(message)
}, 1000) }, 1000)
while (same_levels.length) { while (same_levels.length) {
await Promise.all(same_levels.map(async v => { const same_levels_missed = same_levels.filter(v => !mapping[v.id])
await Promise.all(same_levels_missed.map(async v => {
try { try {
const { name, id, parent } = v const { name, id, parent } = v
const target = mapping[parent] || root const target = mapping[parent] || root
@ -602,16 +599,16 @@ async function create_folders ({ source, old_mapping, folders, root, task_id, se
count++ count++
mapping[id] = new_folder.id mapping[id] = new_folder.id
const mapping_record = id + ' ' + new_folder.id + '\n' const mapping_record = id + ' ' + new_folder.id + '\n'
db.prepare('update task set status=?, mapping = mapping || ? where id=?').run('copying', mapping_record, task_id) db.prepare('update task set mapping = mapping || ? where id=?').run(mapping_record, task_id)
} catch (e) { } catch (e) {
if (e.message === FILE_EXCEED_MSG) { if (e.message === FILE_EXCEED_MSG) {
clearInterval(loop) clearInterval(loop)
throw new Error(FILE_EXCEED_MSG) throw new Error(FILE_EXCEED_MSG)
} }
console.error('创建目录出错:', e.message) console.error('創建目錄出錯:', e.message)
} }
})) }))
folders = folders.filter(v => !mapping[v.id]) // folders = folders.filter(v => !mapping[v.id])
same_levels = [].concat(...same_levels.map(v => folders.filter(vv => vv.parent === v.id))) same_levels = [].concat(...same_levels.map(v => folders.filter(vv => vv.parent === v.id)))
} }
@ -659,10 +656,10 @@ async function confirm_dedupe ({ file_number, folder_number }) {
const answer = await prompts({ const answer = await prompts({
type: 'select', type: 'select',
name: 'value', name: 'value',
message: `检测到同位置下重复文件${file_number}个,重复空目录${folder_number}个,是否删除?`, message: `檢測到同位置下重複文件${file_number}个,重複空目錄${folder_number}個,是否刪除?`,
choices: [ choices: [
{ title: 'Yes', description: '确认删除', value: 'yes' }, { title: 'Yes', description: '確認刪除', value: 'yes' },
{ title: 'No', description: '先不除', value: 'no' } { title: 'No', description: '先不除', value: 'no' }
], ],
initial: 0 initial: 0
}) })
@ -687,7 +684,7 @@ async function rm_file ({ fid, service_account }) {
} catch (err) { } catch (err) {
retry++ retry++
handle_error(err) handle_error(err)
console.log('删除重试中,重试次数', retry) console.log('刪除重試中,重試次數', retry)
} }
} }
} }
@ -697,7 +694,7 @@ async function dedupe ({ fid, update, service_account }) {
if (!update) { if (!update) {
const info = get_all_by_fid(fid) const info = get_all_by_fid(fid)
if (info) { if (info) {
console.log('找到本地缓存数据,缓存时间', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss')) console.log('找到本地快取資料,快取時間', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss'))
arr = info arr = info
} }
} }
@ -718,14 +715,14 @@ async function dedupe ({ fid, update, service_account }) {
try { try {
await limit(() => trash_file({ fid: v.id, service_account })) await limit(() => trash_file({ fid: v.id, service_account }))
if (v.mimeType === FOLDER_TYPE) { if (v.mimeType === FOLDER_TYPE) {
console.log('成功删除文件夹', v.name) console.log('成功刪除資料夾', v.name)
folder_count++ folder_count++
} else { } else {
console.log('成功删除文件', v.name) console.log('成功刪除檔案', v.name)
file_count++ file_count++
} }
} catch (e) { } catch (e) {
console.log('删除失败', e.message) console.log('刪除失敗', e.message)
} }
})) }))
return { file_count, folder_count } return { file_count, folder_count }
@ -743,7 +740,7 @@ function handle_error (err) {
function print_progress (msg) { function print_progress (msg) {
if (process.stdout.cursorTo) { if (process.stdout.cursorTo) {
process.stdout.cursorTo(0) process.stdout.cursorTo(0)
process.stdout.write(msg) process.stdout.write(msg + ' ')
} else { } else {
console.log(msg) console.log(msg)
} }

View File

@ -2,16 +2,17 @@ const Router = require('@koa/router')
const { db } = require('../db') const { db } = require('../db')
const { validate_fid, gen_count_body } = require('./gd') const { validate_fid, gen_count_body } = require('./gd')
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks } = require('./tg') const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg')
const { AUTH, ROUTER_PASSKEY, TG_IPLIST, COPY_TARGET2, COPY_TARGET3 } = require('../config') const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config')
const { tg_whitelist } = AUTH const { tg_whitelist } = AUTH
const COPYING_FIDS = {}
const counting = {} const counting = {}
const router = new Router() const router = new Router()
router.get('/api/gdurl/count', async ctx => { router.get('/api/gdurl/count', async ctx => {
if (!ROUTER_PASSKEY) return ctx.body = 'gd-utils 成功啟動' if (!ROUTER_PASSKEY) return ctx.body = 'gd-utils-cht 成功啟動'
const { query, headers } = ctx.request const { query, headers } = ctx.request
let { fid, type, update, passkey } = query let { fid, type, update, passkey } = query
if (passkey !== ROUTER_PASSKEY) return ctx.body = 'invalid passkey' if (passkey !== ROUTER_PASSKEY) return ctx.body = 'invalid passkey'
@ -50,7 +51,7 @@ router.post('/api/gdurl/tgbot', async ctx => {
if (callback_query) { if (callback_query) {
const { id, data } = callback_query const { id, data } = callback_query
const chat_id = callback_query.from.id const chat_id = callback_query.from.id
const [action, fid] = data.split(' ') const [action, fid, target] = data.split(' ')
if (action === 'count') { if (action === 'count') {
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' }) if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' })
counting[fid] = true counting[fid] = true
@ -61,19 +62,11 @@ router.post('/api/gdurl/tgbot', async ctx => {
delete counting[fid] delete counting[fid]
}) })
} else if (action === 'copy') { } else if (action === 'copy') {
tg_copy({ fid, chat_id }).then(task_id => { if (COPYING_FIDS[fid]) return sm({ chat_id, text: `正在處理 ${fid} 的複製命令` })
COPYING_FIDS[fid] = true
tg_copy({ fid, target: get_target_by_alias(target), chat_id }).then(task_id => {
task_id && sm({ chat_id, text: `開始複製任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` }) task_id && sm({ chat_id, text: `開始複製任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
}) }).finally(() => COPYING_FIDS[fid] = false)
} else if (action === 'copy2') {
const target = COPY_TARGET2
tg_copy({ fid, target, chat_id }).then(task_id => {
task_id && sm({ chat_id, text: `開始複製任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
})
} else if (action === 'copy3') {
const target = COPY_TARGET3
tg_copy({ fid, target, chat_id }).then(task_id => {
task_id && sm({ chat_id, text: `開始複製任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
})
} }
return reply_cb_query({ id, data }).catch(console.error) return reply_cb_query({ id, data }).catch(console.error)
} }
@ -90,12 +83,26 @@ router.post('/api/gdurl/tgbot', async ctx => {
})) return console.warn('異常請求') })) return console.warn('異常請求')
const fid = extract_fid(text) || extract_from_text(text) const fid = extract_fid(text) || extract_from_text(text)
const no_fid_commands = ['/task', '/help'] const no_fid_commands = ['/task', '/help', '/bm']
if (!no_fid_commands.some(cmd => text.startsWith(cmd)) && !validate_fid(fid)) { if (!no_fid_commands.some(cmd => text.startsWith(cmd)) && !validate_fid(fid)) {
return sm({ chat_id, text: '未辨識到分享ID' }) return sm({ chat_id, text: '未辨識到分享ID' })
} }
if (text.startsWith('/help')) return send_help(chat_id) if (text.startsWith('/help')) return send_help(chat_id)
if (text.startsWith('/count')) { if (text.startsWith('/bm')) {
const [cmd, action, alias, target] = text.split(' ').map(v => v.trim())
if (!action) return send_all_bookmarks(chat_id)
if (action === 'set') {
if (!alias || !target) return sm({ chat_id, text: '標籤名和dstID不能為空' })
if (alias.length > 24) return sm({ chat_id, text: '標籤名請勿超過24个英文字符' })
if (!validate_fid(target)) return sm({ chat_id, text: 'dstID格式錯誤' })
set_bookmark({ chat_id, alias, target })
} else if (action === 'unset') {
if (!alias) return sm({ chat_id, text: '標籤名不能為空' })
unset_bookmark({ chat_id, alias })
} else {
send_bm_help(chat_id)
}
} else if (text.startsWith('/count')) {
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' }) if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' })
try { try {
counting[fid] = true counting[fid] = true
@ -108,7 +115,8 @@ router.post('/api/gdurl/tgbot', async ctx => {
delete counting[fid] delete counting[fid]
} }
} else if (text.startsWith('/copy')) { } else if (text.startsWith('/copy')) {
const target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim())[1] let target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim())[1]
target = get_target_by_alias(target) || target
if (target && !validate_fid(target)) return sm({ chat_id, text: `目標ID ${target} 格式不正確` }) if (target && !validate_fid(target)) return sm({ chat_id, text: `目標ID ${target} 格式不正確` })
const update = text.endsWith(' -u') const update = text.endsWith(' -u')
tg_copy({ fid, target, chat_id, update }).then(task_id => { tg_copy({ fid, target, chat_id, update }).then(task_id => {

View File

@ -26,7 +26,7 @@ function make_table ({ file_count, folder_count, total_size, details }) {
return arr.map(content => ({ content, hAlign })) return arr.map(content => ({ content, hAlign }))
}) })
const total_count = file_count + folder_count const total_count = file_count + folder_count
const tails = ['总计', total_count, total_size].map(v => ({ content: colors.bold(v), hAlign })) const tails = ['總計', total_count, total_size].map(v => ({ content: colors.bold(v), hAlign }))
tb.push(headers, ...records) tb.push(headers, ...records)
tb.push(tails) tb.push(tails)
return tb.toString() + '\n' return tb.toString() + '\n'

131
src/tg.js
View File

@ -5,7 +5,7 @@ const HttpsProxyAgent = require('https-proxy-agent')
const { db } = require('../db') const { db } = require('../db')
const { gen_count_body, validate_fid, real_copy, get_name_by_id } = require('./gd') const { gen_count_body, validate_fid, real_copy, get_name_by_id } = require('./gd')
const { AUTH, DEFAULT_TARGET } = require('../config') const { AUTH, DEFAULT_TARGET, USE_PERSONAL_AUTH } = require('../config')
const { tg_token } = AUTH const { tg_token } = AUTH
const gen_link = (fid, text) => `<a href="https://drive.google.com/drive/folders/${fid}">${text || fid}</a>` const gen_link = (fid, text) => `<a href="https://drive.google.com/drive/folders/${fid}">${text || fid}</a>`
@ -22,24 +22,75 @@ async function get_folder_name (fid) {
return FID_TO_NAME[fid] = name return FID_TO_NAME[fid] = name
} }
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text }
function send_help (chat_id) { function send_help (chat_id) {
const text = `<pre>[使用說明] const text = `<pre>[使用說明]
***不支持單檔分享*** ***不支持單檔分享***
命令 說明 命令 說明
=====================
/help | 使 /help | 使
=====================
/count sourceID [-u] | sourceID, sourceIDID -u /count sourceID [-u] | sourceID
sourceID可以是共享網址本身也可以是共享ID如果命令最后加上 -u則無視快取記錄強制從線上獲取適合一段時候後才更新完畢的分享連結
/copy sourceID targetID() [-u] | sourceIDtargetIDtargetIDconfig.jsDEFAULT_TARGET -utaskID =====================
/copy sourceID targetID() [-u] | sourceIDtargetID
/task taskID() | taskID all () 若無targetID則會複製到預設位置config.js中的DEFAULT_TARGET
如果設定了bookmark那麼targetID也可以是bookmark的標籤名
如果命令最後加上 -u則無視快取記錄強制從線上獲取源資料夾資訊返回拷貝任務的taskID
=====================
/task taskID() | taskID
若填 all 則返回所有任務列表(歷史紀錄)
=====================
/bm [action] [alias] [target] | bookmarkID
會在輸入共享連結後返回的文件統計開始複製這兩個按鈕的下方出現方便複製到常用位置
範例
/bm |
/bm set movie folder-id | folder-idmovie
/bm unset movie |
</pre>` </pre>`
return sm({ chat_id, text, parse_mode: 'HTML' }) return sm({ chat_id, text, parse_mode: 'HTML' })
} }
function send_bm_help (chat_id) {
const text = `<pre>/bm [action] [alias] [target] | bookmark添加常用目的資料夾ID
會在輸入共享連結後返回的文件統計開始複製這兩個按鈕的下方出現方便複製到常用位置
範例
/bm |
/bm set movie folder-id | folder-idmovie
/bm unset movie |
</pre>`
return sm({ chat_id, text, parse_mode: 'HTML' })
}
function send_all_bookmarks (chat_id) {
let records = db.prepare('select alias, target from bookmark').all()
if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
const tb = new Table({ style: { head: [], border: [] } })
const headers = ['標籤名', 'dstID']
records = records.map(v => [v.alias, v.target])
tb.push(headers, ...records)
const text = tb.toString().replace(/─/g, '—')
return sm({ chat_id, text: `<pre>${text}</pre>`, parse_mode: 'HTML' })
}
function set_bookmark ({ chat_id, alias, target }) {
const record = db.prepare('select alias from bookmark where alias=?').get(alias)
if (record) return sm({ chat_id, text: '資料庫中已有同名的收藏' })
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
return sm({ chat_id, text: `成功設定收藏${alias} | ${target}` })
}
function unset_bookmark ({ chat_id, alias }) {
const record = db.prepare('select alias from bookmark where alias=?').get(alias)
if (!record) return sm({ chat_id, text: '未找到此標籤名的收藏' })
db.prepare('delete from bookmark where alias=?').run(alias)
return sm({ chat_id, text: '成功刪除收藏 ' + alias })
}
function get_target_by_alias (alias) {
const record = db.prepare('select target from bookmark where alias=?').get(alias)
return record && record.target
}
function send_choice ({ fid, chat_id }) { function send_choice ({ fid, chat_id }) {
return sm({ return sm({
chat_id, chat_id,
@ -47,22 +98,27 @@ function send_choice ({ fid, chat_id }) {
reply_markup: { reply_markup: {
inline_keyboard: [ inline_keyboard: [
[ [
{ text: '文件統計', callback_data: `count ${fid}` } { text: '文件統計', callback_data: `count ${fid}` },
], { text: '開始複製', callback_data: `copy ${fid}` }
[
{ text: '開始複製(預設)', callback_data: `copy ${fid}` }
],
[
{ text: '開始複製(1)', callback_data: `copy2 ${fid}` }
],
[
{ text: '開始複製(2)', callback_data: `copy3 ${fid}` }
] ]
] ].concat(gen_bookmark_choices(fid))
} }
}) })
} }
// console.log(gen_bookmark_choices())
function gen_bookmark_choices (fid) {
const gen_choice = v => ({text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}`})
const records = db.prepare('select * from bookmark').all()
const result = []
for (let i = 0; i < records.length; i += 2) {
const line = [gen_choice(records[i])]
if (records[i + 1]) line.push(gen_choice(records[i + 1]))
result.push(line)
}
return result
}
async function send_all_tasks (chat_id) { async function send_all_tasks (chat_id) {
let records = db.prepare('select id, status, ctime from task').all() let records = db.prepare('select id, status, ctime from task').all()
if (!records.length) return sm({ chat_id, text: '資料庫中沒有任務記錄' }) if (!records.length) return sm({ chat_id, text: '資料庫中沒有任務記錄' })
@ -120,7 +176,7 @@ async function get_task_info (task_id) {
} }
async function send_task_info ({ task_id, chat_id }) { async function send_task_info ({ task_id, chat_id }) {
const { text, status, total_count } = await get_task_info(task_id) const { text, status, folder_count } = await get_task_info(task_id)
if (!text) return sm({ chat_id, text: '資料庫查無此任務ID' + task_id }) if (!text) return sm({ chat_id, text: '資料庫查無此任務ID' + task_id })
const url = `https://api.telegram.org/bot${tg_token}/sendMessage` const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
let message_id let message_id
@ -130,8 +186,8 @@ async function send_task_info ({ task_id, chat_id }) {
} catch (e) { } catch (e) {
console.log('fail to send message to tg', e.message) console.log('fail to send message to tg', e.message)
} }
// get_task_info 在task文件数超大时比较吃cpu如果超5万就不每10秒更新了 // get_task_info 在task目录数超大时比较吃cpu如果超1万就不每10秒更新了以后如果把mapping 也另存一张表可以取消此限制
if (!message_id || status !== 'copying' || total_count > 50000) return if (!message_id || status !== 'copying' || folder_count > 10000) return
const loop = setInterval(async () => { const loop = setInterval(async () => {
const url = `https://api.telegram.org/bot${tg_token}/editMessageText` const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
const { text, status } = await get_task_info(task_id) const { text, status } = await get_task_info(task_id)
@ -150,31 +206,19 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
let record = db.prepare('select id, status from task where source=? and target=?').get(fid, target) let record = db.prepare('select id, status from task where source=? and target=?').get(fid, target)
if (record) { if (record) {
if (record.status === 'copying') { if (record.status === 'copying') {
sm({ chat_id, text: '已有相同源ID和目的ID的任務正在進行查詢進度可輸入 /task ' + record.id }) sm({ chat_id, text: '已有相同源ID和目的ID的任務正在進行查詢進度可輸入 /task ' + record.id })
return return
} else if (record.status === 'finished') { } else if (record.status === 'finished') {
sm({ chat_id, text: `檢測到已存在的任務 ${record.id},開始繼續拷貝` }) sm({ chat_id, text: `檢測到已存在的任務 ${record.id},開始繼續拷貝` })
} }
} }
real_copy({ source: fid, update, target, not_teamdrive: true, service_account: true, is_server: true }) real_copy({ source: fid, update, target, service_account: !USE_PERSONAL_AUTH, is_server: true })
.then(async info => { .then(async info => {
if (!record) record = {} // 防止无限循环 if (!record) record = {} // 防止无限循环
if (!info) return if (!info) return
const { task_id } = info const { task_id } = info
const row = db.prepare('select * from task where id=?').get(task_id) const { text } = await get_task_info(task_id)
const { source, target, status, copied, mapping, ctime, ftime } = row
const { summary } = db.prepare('select summary from gd where fid=?').get(source) || {}
const { file_count, folder_count, total_size } = summary ? JSON.parse(summary) : {}
const copied_files = copied ? copied.trim().split('\n').length : 0
const copied_folders = mapping ? (mapping.trim().split('\n').length - 1) : 0
let text = `任務 ${task_id} 完成\n`
const name = await get_folder_name(source)
text += '源資料夾:' + gen_link(source, name) + '\n'
text += '目錄完成數:' + copied_folders + '/' + folder_count + '\n'
text += '文件完成數:' + copied_files + '/' + file_count + '\n'
text += '合計大小:' + (total_size || '未知大小') + '\n'
sm({ chat_id, text, parse_mode: 'HTML' }) sm({ chat_id, text, parse_mode: 'HTML' })
}) })
.catch(err => { .catch(err => {
@ -206,7 +250,8 @@ function reply_cb_query ({ id, data }) {
} }
async function send_count ({ fid, chat_id, update }) { async function send_count ({ fid, chat_id, update }) {
const table = await gen_count_body({ fid, update, type: 'tg', service_account: true }) sm({ chat_id, text: `開始獲取 ${fid} 所有檔案資訊,請稍後,建議統計完成前先不要開始複製,因为複製也需要先獲取來源資料夾資訊` })
const table = await gen_count_body({ fid, update, type: 'tg', service_account: !USE_PERSONAL_AUTH })
if (!table) return sm({ chat_id, parse_mode: 'HTML', text: gen_link(fid) + ' 資訊獲取失敗' }) if (!table) return sm({ chat_id, parse_mode: 'HTML', text: gen_link(fid) + ' 資訊獲取失敗' })
const url = `https://api.telegram.org/bot${tg_token}/sendMessage` const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
const gd_link = `https://drive.google.com/drive/folders/${fid}` const gd_link = `https://drive.google.com/drive/folders/${fid}`
@ -222,7 +267,7 @@ ${table}</pre>`
// const too_long_msgs = ['request entity too large', 'message is too long'] // const too_long_msgs = ['request entity too large', 'message is too long']
// if (description && too_long_msgs.some(v => description.toLowerCase().includes(v))) { // if (description && too_long_msgs.some(v => description.toLowerCase().includes(v))) {
if (true) { if (true) {
const smy = await gen_count_body({ fid, type: 'json', service_account: true }) const smy = await gen_count_body({ fid, type: 'json', service_account: !USE_PERSONAL_AUTH })
const { file_count, folder_count, total_size } = JSON.parse(smy) const { file_count, folder_count, total_size } = JSON.parse(smy)
return sm({ return sm({
chat_id, chat_id,
@ -256,7 +301,7 @@ function extract_fid (text) {
if (!text.startsWith('http')) text = 'https://' + text if (!text.startsWith('http')) text = 'https://' + text
const u = new URL(text) const u = new URL(text)
if (u.pathname.includes('/folders/')) { if (u.pathname.includes('/folders/')) {
const reg = /[^\/?]+$/ const reg = /[^/?]+$/
const match = u.pathname.match(reg) const match = u.pathname.match(reg)
return match && match[0] return match && match[0]
} }
@ -271,3 +316,5 @@ function extract_from_text (text) {
const m = text.match(reg) const m = text.match(reg)
return m && extract_fid(m[0]) return m && extract_fid(m[0])
} }
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark }