bughz
/
gd-utils
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

just update

This commit is contained in:
liaojack8 2020-07-30 17:43:16 +08:00
parent a131529bc1
commit 134a79f808
11 changed files with 135 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
backup/.keep Normal file
View File

28
bookmark.js Normal file
View File

@ -0,0 +1,28 @@
const fs = require('fs')
const {db} = require('./db')
const action = process.argv[2] || 'export'
const filepath = process.argv[3] || 'bookmarks.json'
if (action === 'export') {
const bookmarks = db.prepare('select * from bookmark').all()
fs.writeFileSync(filepath, JSON.stringify(bookmarks))
console.log('bookmarks exported', filepath)
} else if (action === 'import') {
let bookmarks = fs.readFileSync(filepath, 'utf8')
bookmarks = JSON.parse(bookmarks)
bookmarks.forEach(v => {
const {alias, target} = v
const exist = db.prepare('select alias from bookmark where alias=?').get(alias)
if (exist) {
db.prepare('update bookmark set target=? where alias=?').run(target, alias)
} else {
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
}
})
console.log('bookmarks imported', bookmarks)
} else {
console.log('[help info]')
console.log('export: node bookmark.js export bm.json')
console.log('import: node bookmark.js import bm.json')
}

8
clear-db.js Normal file
View File

@ -0,0 +1,8 @@
const { db } = require('./db')
const record = db.prepare('select count(*) as c from gd').get()
db.prepare('delete from gd').run()
console.log('已刪除', record.c, '項資料')
db.exec('vacuum')
db.close()

2
copy
View File

@ -6,6 +6,8 @@ const { argv } = require('yargs')
.usage('用法: ./$0 <source id> <target id> [options]\ntarget id可选，不填则使用config.js里的DEFAULT_TARGET')
.alias('u', 'update')
.describe('u', '不使用本地快取，則無視快取記錄強制從線上獲取源資料夾資訊')
.alias('y', 'yes')
.describe('yes', '如果發現拷貝紀錄，直接繼續上次的進度')
.alias('f', 'file')
.describe('f', '複製單一文件')
.alias('n', 'name')

2
dedupe
View File

@ -2,6 +2,8 @@
const { argv } = require('yargs')
.usage('用法: ./$0 <folder-id> [options]')
.alias('y', 'yes')
.describe('yes', '如果發現重複項目，直接刪除')
.alias('u', 'update')
.describe('u', '不使用本地快取，則無視快取記錄強制從線上獲取源資料夾資訊')
.alias('S', 'service_account')

31
src/gd.js
View File

@ -38,8 +38,9 @@ if (proxy_url) {
axins = axios.create({})
}
const SA_LOCATION = argv.sa || 'sa'
const SA_BATCH_SIZE = 1000
const SA_FILES = fs.readdirSync(path.join(__dirname, '../sa')).filter(v => v.endsWith('.json'))
const SA_FILES = fs.readdirSync(path.join(__dirname, SA_PATH)).filter(v => v.endsWith('.json'))
SA_FILES.flag = 0
let SA_TOKENS = get_sa_batch()
@ -61,7 +62,7 @@ function get_sa_batch () {
SA_FILES.flag = new_flag
return files.map(filename => {
const gtoken = new GoogleToken({
keyFile: path.join(__dirname, '../sa', filename),
keyFile: path.join(__dirname, SA_PATH, filename),
scope: ['https://www.googleapis.com/auth/drive']
})
return { gtoken, expires: 0 }
@ -197,7 +198,7 @@ function get_all_by_fid (fid) {
}
async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
const result = []
let result = []
const not_finished = []
const limit = pLimit(PARALLEL_LIMIT)
@ -226,7 +227,7 @@ async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
should_save && save_files_to_db(parent, files)
const folders = files.filter(v => v.mimeType === FOLDER_TYPE)
files.forEach(v => v.parent = parent)
result.push(...files)
result = result.concat(files)
return Promise.all(folders.map(v => recur(v.id)))
}
try {
@ -268,15 +269,21 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
params.pageSize = Math.min(PAGE_SIZE, 1000)
// const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa
const use_sa = (fid !== 'root') && service_account
const headers = await gen_headers(use_sa)
// const headers = await gen_headers(use_sa)
// 对于直接子文件数超多的目录（1ctMwpIaBg8S1lrZDxdynLXJpMsm5guAl），可能还没列完，access_token就过期了
// 由于需要nextPageToken才能获取下一页的数据，所以无法用并行请求，测试发现每次获取1000个文件的请求大多需要20秒以上才能完成
const gtoken = use_sa && (await get_sa_token()).gtoken
do {
if (pageToken) params.pageToken = pageToken
let url = 'https://www.googleapis.com/drive/v3/files'
url += '?' + params_to_query(params)
const payload = { headers, timeout: TIMEOUT_BASE }
let retry = 0
let data
const payload = { timeout: TIMEOUT_BASE }
while (!data && (retry < RETRY_LIMIT)) {
const access_token = gtoken ? (await gtoken.getToken()).access_token : (await get_access_token())
const headers = { authorization: 'Bearer ' + access_token }
payload.headers = headers
try {
data = (await axins.get(url, payload)).data
} catch (err) {
@ -291,6 +298,7 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
return files
}
files = files.concat(data.files)
argv.sfl && console.log('files.length:', files.length)
pageToken = data.nextPageToken
} while (pageToken)
@ -466,7 +474,7 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
const record = db.prepare('select * from task where source=? and target=?').get(source, target)
if (record) {
const copied = db.prepare('select fileid from copied where taskid=?').all(record.id).map(v => v.fileid)
const choice = is_server ? 'continue' : await user_choose()
const choice = (is_server || argv.yes) ? 'continue' : await user_choose()
if (choice === 'exit') {
return console.log('退出程序')
} else if (choice === 'continue') {
@ -561,6 +569,7 @@ async function copy_files ({ files, mapping, service_account, root, task_id }) {
do {
if (err) {
clearInterval(loop)
files = null
throw err
}
if (concurrency > PARALLEL_LIMIT) {
@ -632,7 +641,7 @@ async function copy_file (id, parent, use_sa, limit, task_id) {
if (message && message.toLowerCase().includes('file limit')) {
if (limit) limit.clearQueue()
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
throw new Error('您的小組雲端硬碟文件數已超限，停止複製')
throw new Error(FILE_EXCEED_MSG)
}
if (use_sa && message && message.toLowerCase().includes('rate limit')) {
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken)
@ -784,7 +793,7 @@ async function rm_file ({ fid, service_account }) {
}
}
async function dedupe ({ fid, update, service_account }) {
async function dedupe ({ fid, update, service_account, yes }) {
let arr
if (!update) {
const info = get_all_by_fid(fid)
@ -797,7 +806,7 @@ async function dedupe ({ fid, update, service_account }) {
const dupes = find_dupe(arr)
const folder_number = dupes.filter(v => v.mimeType === FOLDER_TYPE).length
const file_number = dupes.length - folder_number
const choice = await confirm_dedupe({ file_number, folder_number })
const choice = yes || await confirm_dedupe({ file_number, folder_number })
if (choice === 'no') {
return console.log('退出程序')
} else if (!choice) {
@ -842,4 +851,4 @@ function print_progress (msg) {
}
}
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id, get_info_by_id }
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id, get_info_by_id, get_access_token, get_sa_token, walk_and_save }

33
src/router.js
View File

@ -42,14 +42,15 @@ router.get('/api/gdurl/count', async ctx => {
router.post('/api/gdurl/tgbot', async ctx => {
const { body } = ctx.request
console.log('ctx.ip', ctx.ip) // 可以只允许tg服务器的ip
console.log('tg message:', body)
console.log('tg message:', JSON.stringify(body, null, ' '))
if (TG_IPLIST && !TG_IPLIST.includes(ctx.ip)) return ctx.body = 'invalid ip'
ctx.body = '' // 早点释放连接
const message = body.message || body.edited_message
const message_str = JSON.stringify(message)
const { callback_query } = body
if (callback_query) {
const { id, data } = callback_query
const { id, message, data } = callback_query
const chat_id = callback_query.from.id
const [action, fid, target] = data.split(' ').filter(v => v)
if (action === 'count') {
@ -67,22 +68,34 @@ router.post('/api/gdurl/tgbot', async ctx => {
tg_copy({ fid, target: get_target_by_alias(target), chat_id }).then(task_id => {
task_id && sm({ chat_id, text: `開始複製，任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
}).finally(() => COPYING_FIDS[fid] = false)
} else if (action === 'update') {
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計，請稍等片刻' })
counting[fid] = true
send_count({ fid, chat_id, update: true }).finally(() => {
delete counting[fid]
})
} else if (action === 'clear_button') {
const { message_id, text } = message || {}
if (message_id) sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
}
return reply_cb_query({ id, data }).catch(console.error)
}
const chat_id = message && message.chat && message.chat.id
const text = message && message.text && message.text.trim()
const text = (message && message.text && message.text.trim()) || ''
let username = message && message.from && message.from.username
username = username && String(username).toLowerCase()
let user_id = message && message.from && message.from.id
user_id = user_id && String(user_id).toLowerCase()
if (!chat_id || !text || !tg_whitelist.some(v => {
if (!chat_id || !tg_whitelist.some(v => {
v = String(v).toLowerCase()
return v === username || v === user_id
})) return console.warn('異常請求')
})) {
chat_id && sm({ chat_id, text: '您的使用者名稱或ID不在機器人的白名單中，如果是您配置的機器人，請先到config.js中配置自己的username' })
return console.warn('收到非白名單用戶的請求')
}
const fid = extract_fid(text) || extract_from_text(text)
const fid = extract_fid(text) || extract_from_text(text) || extract_from_text(message_str)
const no_fid_commands = ['/task', '/help', '/bm']
if (!no_fid_commands.some(cmd => text.startsWith(cmd)) && !validate_fid(fid)) {
return sm({ chat_id, text: '未辨識到分享ID' })
@ -93,7 +106,7 @@ router.post('/api/gdurl/tgbot', async ctx => {
if (!action) return send_all_bookmarks(chat_id)
if (action === 'set') {
if (!alias || !target) return sm({ chat_id, text: '標籤名和dstID不能為空' })
if (alias.length > 24) return sm({ chat_id, text: '標籤名請勿超過24个英文字符' })
if (alias.length > 24) return sm({ chat_id, text: '標籤名不要超過24個英文字符長度' })
if (!validate_fid(target)) return sm({ chat_id, text: 'dstID格式錯誤' })
set_bookmark({ chat_id, alias, target })
} else if (action === 'unset') {
@ -143,10 +156,10 @@ router.post('/api/gdurl/tgbot', async ctx => {
return running_tasks.forEach(v => send_task_info({ chat_id, task_id: v.id }).catch(console.error))
}
send_task_info({ task_id, chat_id }).catch(console.error)
} else if (text.includes('drive.google.com/') || validate_fid(text)) {
return send_choice({ fid: fid || text, chat_id }).catch(console.error)
} else if (message_str.includes('drive.google.com/') || validate_fid(text)) {
return send_choice({ fid: fid || text, chat_id })
} else {
sm({ chat_id, text: '暫不支持此命令' })
sm({ chat_id, text: '暫不支援此命令' })
}
})

14
src/summary.js
View File

@ -5,10 +5,10 @@ const { escape } = require('html-escaper')
module.exports = { make_table, summary, make_html, make_tg_table }
function make_html ({ file_count, folder_count, total_size, details }) {
const head = ['類型', '數量', '大小']
const head = ['类型', '数量', '大小']
const th = '<tr>' + head.map(k => `<th>${k}</th>`).join('') + '</tr>'
const td = details.map(v => '<tr>' + [escape(v.ext), v.count, v.size].map(k => `<td>${k}</td>`).join('') + '</tr>').join('')
let tail = ['合計', file_count + folder_count, total_size]
let tail = ['合计', file_count + folder_count, total_size]
tail = '<tr style="font-weight:bold">' + tail.map(k => `<td>${k}</td>`).join('') + '</tr>'
const table = `<table border="1" cellpadding="12" style="border-collapse:collapse;font-family:serif;font-size:22px;margin:10px auto;text-align: center">
${th}
@ -26,7 +26,7 @@ function make_table ({ file_count, folder_count, total_size, details }) {
return arr.map(content => ({ content, hAlign }))
})
const total_count = file_count + folder_count
const tails = ['總計', total_count, total_size].map(v => ({ content: colors.bold(v), hAlign }))
const tails = ['总计', total_count, total_size].map(v => ({ content: colors.bold(v), hAlign }))
tb.push(headers, ...records)
tb.push(tails)
return tb.toString() + '\n'
@ -56,8 +56,8 @@ function make_tg_table ({ file_count, folder_count, total_size, details }) {
const hAlign = 'center'
const headers = ['Type', 'Count', 'Size'].map(v => ({ content: v, hAlign }))
details.forEach(v => {
if (v.ext === '資料夾') v.ext = '[Folder]'
if (v.ext === '無副檔名') v.ext = '[NoExt]'
if (v.ext === '文件夹') v.ext = '[Folder]'
if (v.ext === '无扩展名') v.ext = '[NoExt]'
})
const records = details.map(v => [v.ext, v.count, v.size]).map(arr => arr.map(content => ({ content, hAlign })))
const total_count = file_count + folder_count
@ -107,8 +107,8 @@ function summary (info, sort_by) {
} else {
details.sort((a, b) => b.count - a.count)
}
if (no_ext) details.push({ ext: '無副檔名', count: no_ext, size: format_size(no_ext_size), raw_size: no_ext_size })
if (folder_count) details.push({ ext: '資料夾', count: folder_count, size: 0, raw_size: 0 })
if (no_ext) details.push({ ext: '无扩展名', count: no_ext, size: format_size(no_ext_size), raw_size: no_ext_size })
if (folder_count) details.push({ ext: '文件夹', count: folder_count, size: 0, raw_size: 0 })
return { file_count, folder_count, total_size, details }
}

50
src/tg.js
View File

@ -10,7 +10,7 @@ const { BUTTON_LEVEL } = require('../config_mod')
const { tg_token } = AUTH
const gen_link = (fid, text) => `<a href="https://drive.google.com/drive/folders/${fid}">${text || fid}</a>`
if (!tg_token) throw new Error('请先在config.js里设置tg_token')
if (!tg_token) throw new Error('請先在config.js裡設定tg_token')
const { https_proxy } = process.env
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
@ -86,15 +86,15 @@ function clear_tasks (chat_id) {
function rm_task ({ task_id, chat_id }) {
const exist = db.prepare('select id from task where id=?').get(task_id)
if (!exist) return sm({ chat_id, text: `不存在任務ID為 ${task_id} 的任務紀錄` })
if (!exist) return sm({ chat_id, text: `不存在任務ID為 ${task_id} 的任務記錄` })
db.prepare('delete from task where id=?').run(task_id)
db.prepare('delete from copied where taskid=?').run(task_id)
if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 紀錄` })
if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 記錄` })
}
function send_all_bookmarks (chat_id) {
let records = db.prepare('select alias, target from bookmark').all()
if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
if (!records.length) return sm({ chat_id, text: '數據庫中沒有收藏記錄' })
const tb = new Table({ style: { head: [], border: [] } })
const headers = ['標籤名', 'dstID']
records = records.map(v => [v.alias, v.target])
@ -107,7 +107,7 @@ function set_bookmark ({ chat_id, alias, target }) {
const record = db.prepare('select alias from bookmark where alias=?').get(alias)
if (record) return sm({ chat_id, text: '資料庫中已有同名的收藏' })
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
return sm({ chat_id, text: `成功設定收藏${alias} | ${target}` })
return sm({ chat_id, text: `成功設定收藏：${alias} | ${target}` })
}
function unset_bookmark ({ chat_id, alias }) {
@ -139,6 +139,10 @@ function send_choice ({ fid, chat_id }) {
],
[
{ text: '開始複製', callback_data: `copy ${fid}` }
],
[
{ text: '強制更新', callback_data: `update ${fid}` },
{ text: '清除', callback_data: `clear_button` }
]
].concat(gen_bookmark_choices(fid))
}
@ -152,6 +156,10 @@ function send_choice ({ fid, chat_id }) {
[
{ text: '文件統計', callback_data: `count ${fid}` },
{ text: '開始複製', callback_data: `copy ${fid}` }
],
[
{ text: '強制更新', callback_data: `update ${fid}` },
{ text: '清除', callback_data: `clear_button` }
]
].concat(gen_bookmark_choices(fid))
}
@ -167,15 +175,12 @@ function gen_bookmark_choices (fid) {
}else{
level = BUTTON_LEVEL
}
const gen_choice = v => ({text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}`})
const gen_choice = v => ({ text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}` })
const records = db.prepare('select * from bookmark').all()
const result = []
for (let i = 0; i < records.length; i++) {
for (let i = 0; i < records.length; i += 2) {
const line = [gen_choice(records[i])]
for(let j = 0; j < level-1; j ++){
if (records[i+1]) line.push(gen_choice(records[i+1]))
i++
}
if (records[i + 1]) line.push(gen_choice(records[i + 1]))
result.push(line)
}
return result
@ -201,8 +206,8 @@ async function send_all_tasks (chat_id) {
// const description = err.response && err.response.data && err.response.data.description
// if (description && description.includes('message is too long')) {
if (true) {
const text = [headers].concat(records).map(v => v.join('\t')).join('\n')
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務：\n<pre>${text}</pre>` })
const text = [headers].concat(records.slice(-100)).map(v => v.join('\t')).join('\n')
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務(僅顯示最近100項)：\n<pre>${text}</pre>` })
}
console.error(err)
})
@ -247,10 +252,9 @@ async function send_task_info ({ task_id, chat_id }) {
// get_task_info 在task目录数超大时比较吃cpu，以后如果最好把mapping也另存一张表
if (!message_id || status !== 'copying') return
const loop = setInterval(async () => {
const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
const { text, status } = await get_task_info(task_id)
if (status !== 'copying') clearInterval(loop)
axins.post(url, { chat_id, message_id, text, parse_mode: 'HTML' }).catch(e => console.error(e.message))
sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
}, 10 * 1000)
}
@ -281,7 +285,7 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
real_copy({ source: fid, update, target, service_account: !USE_PERSONAL_AUTH, is_server: true })
.then(async info => {
if (!record) record = {} // 防止无限循环
if (!record) record = {} // 防止無限循環
if (!info) return
const { task_id } = info
const { text } = await get_task_info(task_id)
@ -293,7 +297,7 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
if (!record) record = {}
console.error('複製失敗', fid, '-->', target)
console.error(err)
sm({ chat_id, text: '複製失敗，失敗訊息：' + err.message })
sm({ chat_id, text: (task_id || '') + '複製失敗，失敗訊息：' + err.message })
})
while (!record) {
@ -320,7 +324,7 @@ function reply_cb_query ({ id, data }) {
async function send_count ({ fid, chat_id, update }) {
sm({ chat_id, text: `開始獲取 ${fid} 所有檔案資訊，請稍後，建議統計完成前先不要開始複製，因为複製也需要先獲取來源資料夾資訊` })
const table = await gen_count_body({ fid, update, type: 'tg', service_account: !USE_PERSONAL_AUTH })
if (!table) return sm({ chat_id, parse_mode: 'HTML', text: gen_link(fid) + ' 信息获取失败' })
if (!table) return sm({ chat_id, parse_mode: 'HTML', text: gen_link(fid) + ' 資訊獲取失敗' })
const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
const gd_link = `https://drive.google.com/drive/folders/${fid}`
const name = await get_folder_name(fid)
@ -353,11 +357,14 @@ ${table}</pre>`
})
}
function sm (data) {
const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
function sm (data, endpoint) {
endpoint = endpoint || 'sendMessage'
const url = `https://api.telegram.org/bot${tg_token}/${endpoint}`
return axins.post(url, data).catch(err => {
// console.error('fail to post', url, data)
console.error('fail to send message to tg:', err.message)
const err_data = err.response && err.response.data
err_data && console.error(err_data)
})
}
@ -382,7 +389,8 @@ function extract_fid (text) {
}
function extract_from_text (text) {
const reg = /https?:\/\/drive.google.com\/[^\s]+/g
// const reg = /https?:\/\/drive.google.com\/[^\s]+/g
const reg = /https?:\/\/drive.google.com\/[a-zA-Z0-9_\\/?=&-]+/g
const m = text.match(reg)
return m && extract_fid(m[0])
}

4
src/tree.js
View File

@ -39,7 +39,7 @@ function gen_tree_data (data, is_folder) {
const files = data.filter(v => !is_folder(v))
const total_size = sum(files.map(v => v.size))
const root = {
title: `/根目錄 [共${files.length} 個檔案（不包括資料夾）, ${format_size(total_size)}]`,
title: `/根目录 [共${files.length} 个文件（不包括文件夹）, ${format_size(total_size)}]`,
key: data[0].parent
}
if (!folders.length) return [root]
@ -53,7 +53,7 @@ function gen_tree_data (data, is_folder) {
folders.forEach(v => {
let {name, size, count, id} = v
if (name.length > 50) name = name.slice(0, 48) + '...'
v.title = `${name} | [共${count}個檔案 ${format_size(size)}]`
v.title = `${name} | [共${count}个文件 ${format_size(size)}]`
})
root.children = sub_folders.map(v => gen_node(v, folders))
return [root]

14
validate-sa.js
View File

@ -79,17 +79,27 @@ async function get_invalid_sa (arr, fid) {
await get_info(fid, access_token)
good++
} catch (e) {
handle_error(e)
const status = e && e.response && e.response.status
if (Number(status) === 400) fails.push(filename) // access_token 獲取失敗
if (Number(status) === 400) fails.push(filename) // access_token 获取失败
const data = e && e.response && e.response.data
const code = data && data.error && data.error.code
if (Number(code) === 404) fails.push(filename) // 讀取資料夾訊息失敗
if ([404, 403].includes(Number(code))) fails.push(filename) // 读取文件夹信息失败
}
}
return fails
}
function handle_error (err) {
const data = err && err.response && err.response.data
if (data) {
console.error(JSON.stringify(data))
} else {
console.error(err.message)
}
}
async function get_info (fid, access_token) {
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
let params = {