const fs = require('fs') const path = require('path') const dayjs = require('dayjs') const prompts = require('prompts') const pLimit = require('p-limit') const axios = require('@viegg/axios') const HttpsProxyAgent = require('https-proxy-agent') const { GoogleToken } = require('gtoken') const handle_exit = require('signal-exit') const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET, SA_PATH } = require('../config') const { db } = require('../db') const { make_table, make_tg_table, make_html, summary } = require('./summary') const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製' const FOLDER_TYPE = 'application/vnd.google-apps.folder' const { https_proxy } = process.env const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {}) const SA_BATCH_SIZE = 1000 const SA_FILES = fs.readdirSync(path.join(__dirname, SA_PATH)).filter(v => v.endsWith('.json')) SA_FILES.flag = 0 let SA_TOKENS = get_sa_batch() function get_sa_batch () { const new_flag = SA_FILES.flag + SA_BATCH_SIZE const files = SA_FILES.slice(SA_FILES.flag, new_flag) SA_FILES.flag = new_flag return files.map(filename => { const gtoken = new GoogleToken({ keyFile: path.join(__dirname, '../sa', filename), scope: ['https://www.googleapis.com/auth/drive'] }) return { gtoken, expires: 0 } }) } handle_exit(() => { // console.log('handle_exit running') const records = db.prepare('select id from task where status=?').all('copying') records.forEach(v => { db.prepare('update task set status=? where id=?').run('interrupt', v.id) }) records.length && console.log(records.length, 'task interrupted') }) async function gen_count_body ({ fid, type, update, service_account }) { async function update_info () { const info = await walk_and_save({ fid, update, service_account }) // 这一步已经将fid记录存入数据库中了 const row = db.prepare('SELECT summary from gd WHERE fid=?').get(fid) if (!row) return [] return [info, JSON.parse(row.summary)] } function render_smy (smy, type) { if (!smy) return if (['html', 'curl', 'tg'].includes(type)) { smy = (typeof smy === 'object') ? smy : JSON.parse(smy) const type_func = { html: make_html, curl: make_table, tg: make_tg_table } return type_func[type](smy) } else { // 默认输出json return (typeof smy === 'string') ? smy : JSON.stringify(smy) } } let info, smy const record = db.prepare('SELECT * FROM gd WHERE fid = ?').get(fid) if (!record || update) { [info, smy] = await update_info() } if (type === 'all') { info = info || get_all_by_fid(fid) if (!info) { // 说明上次统计过程中断了 [info] = await update_info() } return info && JSON.stringify(info) } if (smy) return render_smy(smy, type) if (record && record.summary) return render_smy(record.summary, type) info = info || get_all_by_fid(fid) if (info) { smy = summary(info) } else { [info, smy] = await update_info() } return render_smy(smy, type) } async function count ({ fid, update, sort, type, output, not_teamdrive, service_account }) { sort = (sort || '').toLowerCase() type = (type || '').toLowerCase() output = (output || '').toLowerCase() if (!update) { const info = get_all_by_fid(fid) if (info) { console.log('找到本地快取資料,快取時間:', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss')) const out_str = get_out_str({ info, type, sort }) if (output) return fs.writeFileSync(output, out_str) return console.log(out_str) } } const result = await walk_and_save({ fid, not_teamdrive, update, service_account }) const out_str = get_out_str({ info: result, type, sort }) if (output) { fs.writeFileSync(output, out_str) } else { console.log(out_str) } } function get_out_str ({ info, type, sort }) { const smy = summary(info, sort) let out_str if (type === 'html') { out_str = make_html(smy) } else if (type === 'json') { out_str = JSON.stringify(smy) } else if (type === 'all') { out_str = JSON.stringify(info) } else { out_str = make_table(smy) } return out_str } function get_all_by_fid (fid) { const record = db.prepare('SELECT * FROM gd WHERE fid = ?').get(fid) if (!record) return null const { info, subf } = record let result = JSON.parse(info) result = result.map(v => { v.parent = fid return v }) if (!subf) return result return recur(result, JSON.parse(subf)) function recur (result, subf) { if (!subf.length) return result const arr = subf.map(v => { const row = db.prepare('SELECT * FROM gd WHERE fid = ?').get(v) if (!row) return null // 如果没找到对应的fid记录,说明上次中断了进程或目录读取未完成 let info = JSON.parse(row.info) info = info.map(vv => { vv.parent = v return vv }) return { info, subf: JSON.parse(row.subf) } }) if (arr.some(v => v === null)) return null const sub_subf = [].concat(...arr.map(v => v.subf).filter(v => v)) result = result.concat(...arr.map(v => v.info)) return recur(result, sub_subf) } } async function walk_and_save ({ fid, not_teamdrive, update, service_account }) { const result = [] const not_finished = [] const limit = pLimit(PARALLEL_LIMIT) const loop = setInterval(() => { const now = dayjs().format('HH:mm:ss') const message = `${now} | 已獲取對象 ${result.length} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}` print_progress(message) }, 1000) async function recur (parent) { let files, should_save if (update) { files = await limit(() => ls_folder({ fid: parent, not_teamdrive, service_account })) should_save = true } else { const record = db.prepare('SELECT * FROM gd WHERE fid = ?').get(parent) if (record) { files = JSON.parse(record.info) } else { files = await limit(() => ls_folder({ fid: parent, not_teamdrive, service_account })) should_save = true } } if (!files) return if (files.not_finished) not_finished.push(parent) should_save && save_files_to_db(parent, files) const folders = files.filter(v => v.mimeType === FOLDER_TYPE) files.forEach(v => v.parent = parent) result.push(...files) return Promise.all(folders.map(v => recur(v.id))) } try { await recur(fid) } catch (e) { console.error(e) } console.log('\n資訊獲取完畢') not_finished.length ? console.log('未讀取完畢的目錄ID:', JSON.stringify(not_finished)) : console.log('所有目錄讀取完畢') clearInterval(loop) const smy = summary(result) db.prepare('UPDATE gd SET summary=?, mtime=? WHERE fid=?').run(JSON.stringify(smy), Date.now(), fid) return result } function save_files_to_db (fid, files) { // 不保存请求未完成的目录,那么下次调用get_all_by_id会返回null,从而再次调用walk_and_save试图完成此目录的请求 if (files.not_finished) return let subf = files.filter(v => v.mimeType === FOLDER_TYPE).map(v => v.id) subf = subf.length ? JSON.stringify(subf) : null const exists = db.prepare('SELECT fid FROM gd WHERE fid = ?').get(fid) if (exists) { db.prepare('UPDATE gd SET info=?, subf=?, mtime=? WHERE fid=?') .run(JSON.stringify(files), subf, Date.now(), fid) } else { db.prepare('INSERT INTO gd (fid, info, subf, ctime) VALUES (?, ?, ?, ?)') .run(fid, JSON.stringify(files), subf, Date.now()) } } async function ls_folder ({ fid, not_teamdrive, service_account }) { let files = [] let pageToken const search_all = { includeItemsFromAllDrives: true, supportsAllDrives: true } const params = ((fid === 'root') || not_teamdrive) ? {} : search_all params.q = `'${fid}' in parents and trashed = false` params.orderBy = 'folder,name desc' params.fields = 'nextPageToken, files(id, name, mimeType, size, md5Checksum)' params.pageSize = Math.min(PAGE_SIZE, 1000) // const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa const use_sa = (fid !== 'root') && service_account const headers = await gen_headers(use_sa) do { if (pageToken) params.pageToken = pageToken let url = 'https://www.googleapis.com/drive/v3/files' url += '?' + params_to_query(params) const payload = { headers, timeout: TIMEOUT_BASE } let retry = 0 let data while (!data && (retry < RETRY_LIMIT)) { try { data = (await axins.get(url, payload)).data } catch (err) { handle_error(err) retry++ payload.timeout = Math.min(payload.timeout * 2, TIMEOUT_MAX) } } if (!data) { console.error('讀取目錄未完成(部分讀取), 參數:', params) files.not_finished = true return files } files = files.concat(data.files) pageToken = data.nextPageToken } while (pageToken) return files } async function gen_headers (use_sa) { // use_sa = use_sa && SA_TOKENS.length const access_token = use_sa ? (await get_sa_token()).access_token : (await get_access_token()) return { authorization: 'Bearer ' + access_token } } function params_to_query (data) { const ret = [] for (let d in data) { ret.push(encodeURIComponent(d) + '=' + encodeURIComponent(data[d])) } return ret.join('&') } async function get_access_token () { const { expires, access_token, client_id, client_secret, refresh_token } = AUTH if (expires > Date.now()) return access_token const url = 'https://www.googleapis.com/oauth2/v4/token' const headers = { 'Content-Type': 'application/x-www-form-urlencoded' } const config = { headers } const params = { client_id, client_secret, refresh_token, grant_type: 'refresh_token' } const { data } = await axins.post(url, params_to_query(params), config) // console.log('Got new token:', data) AUTH.access_token = data.access_token AUTH.expires = Date.now() + 1000 * data.expires_in return data.access_token } // get_sa_token().catch(console.error) async function get_sa_token () { if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch() while (SA_TOKENS.length) { const tk = get_random_element(SA_TOKENS) try { return await real_get_sa_token(tk) } catch (e) { console.log(e) SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== tk.gtoken) if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch() } } throw new Error('沒有可用的SA帳號') } async function real_get_sa_token (el) { const { value, expires, gtoken } = el // 把gtoken传递出去的原因是当某账号流量用尽时可以依此过滤 if (Date.now() < expires) return { access_token: value, gtoken } const { access_token, expires_in } = await gtoken.getToken({ forceRefresh: true }) el.value = access_token el.expires = Date.now() + 1000 * (expires_in - 60 * 5) // 提前5分钟判定为过期 return { access_token, gtoken } } function get_random_element (arr) { return arr[~~(arr.length * Math.random())] } function validate_fid (fid) { if (!fid) return false fid = String(fid) const whitelist = ['root', 'appDataFolder', 'photos'] if (whitelist.includes(fid)) return true if (fid.length < 10 || fid.length > 100) return false const reg = /^[a-zA-Z0-9_-]+$/ return fid.match(reg) } async function create_folder (name, parent, use_sa, limit) { let url = `https://www.googleapis.com/drive/v3/files` const params = { supportsAllDrives: true } url += '?' + params_to_query(params) const post_data = { name, mimeType: FOLDER_TYPE, parents: [parent] } let retry = 0 let err_message while (retry < RETRY_LIMIT) { try { const headers = await gen_headers(use_sa) return (await axins.post(url, post_data, { headers })).data } catch (err) { err_message = err.message retry++ handle_error(err) const data = err && err.response && err.response.data const message = data && data.error && data.error.message if (message && message.toLowerCase().includes('file limit')) { if (limit) limit.clearQueue() throw new Error(FILE_EXCEED_MSG) } console.log('創建目錄重試中:', name, '重試次數:', retry) } } throw new Error(err_message + ' 目錄名:' + name) } async function get_name_by_id (fid) { try { const { name } = await get_info_by_id(fid, true) return name } catch (e) { return fid } } async function get_info_by_id (fid, use_sa) { let url = `https://www.googleapis.com/drive/v3/files/${fid}` let params = { includeItemsFromAllDrives: true, supportsAllDrives: true, corpora: 'allDrives', fields: 'id,name' } url += '?' + params_to_query(params) const headers = await gen_headers(use_sa) const { data } = await axins.get(url, { headers }) return data } async function user_choose () { const answer = await prompts({ type: 'select', name: 'value', message: '檢測到上次的複製紀錄,是否繼續?', choices: [ { title: 'Continue', description: '從上次中斷的地方繼續', value: 'continue' }, { title: 'Restart', description: '無視已存在的紀錄,重新複製', value: 'restart' }, { title: 'Exit', description: '直接退出', value: 'exit' } ], initial: 0 }) return answer.value } async function copy ({ source, target, name, min_size, update, not_teamdrive, service_account, dncnr, is_server }) { target = target || DEFAULT_TARGET if (!target) throw new Error('目標位置不能為空') const record = db.prepare('select id, status from task where source=? and target=?').get(source, target) if (record && record.status === 'copying') return console.log('已有相同來源和目的地的任務正在進行,強制退出') try { return await real_copy({ source, target, name, min_size, update, dncnr, not_teamdrive, service_account, is_server }) } catch (err) { console.error('複製資料夾出錯', err) const record = db.prepare('select id, status from task where source=? and target=?').get(source, target) if (record) db.prepare('update task set status=? where id=?').run('error', record.id) } } // 待解决:如果用户手动ctrl+c中断进程,那么已经发出的请求,就算完成了也不会记录到本地数据库中,所以可能产生重复文件(夹) async function real_copy ({ source, target, name, min_size, update, dncnr, not_teamdrive, service_account, is_server }) { async function get_new_root () { if (dncnr) return { id: target } if (name) { return create_folder(name, target, service_account) } else { const source_info = await get_info_by_id(source, service_account) return create_folder(source_info.name, target, service_account) } } const record = db.prepare('select * from task where source=? and target=?').get(source, target) if (record) { const copied = db.prepare('select fileid from copied where taskid=?').all(record.id).map(v => v.fileid) const choice = is_server ? 'continue' : await user_choose() if (choice === 'exit') { return console.log('退出程序') } else if (choice === 'continue') { let { mapping } = record const old_mapping = {} const copied_ids = {} copied.forEach(id => copied_ids[id] = true) mapping = mapping.trim().split('\n').map(line => line.split(' ')) const root = mapping[0][1] mapping.forEach(arr => old_mapping[arr[0]] = arr[1]) db.prepare('update task set status=? where id=?').run('copying', record.id) const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account }) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE).filter(v => !copied_ids[v.id]) if (min_size) files = files.filter(v => v.size >= min_size) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) console.log('待複製的目錄數:', folders.length) console.log('待複製的檔案數:', files.length) const all_mapping = await create_folders({ old_mapping, source, folders, service_account, root, task_id: record.id }) await copy_files({ files, service_account, root, mapping: all_mapping, task_id: record.id }) db.prepare('update task set status=?, ftime=? where id=?').run('finished', Date.now(), record.id) return { id: root, task_id: record.id } } else if (choice === 'restart') { const new_root = await get_new_root() const root_mapping = source + ' ' + new_root.id + '\n' db.prepare('update task set status=?, mapping=? where id=?').run('copying', root_mapping, record.id) db.prepare('delete from copied where taskid=?').run(record.id) // const arr = await walk_and_save({ fid: source, update: true, not_teamdrive, service_account }) const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account }) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE) if (min_size) files = files.filter(v => v.size >= min_size) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) console.log('待複製的目錄數:', folders.length) console.log('待複製的檔案數:', files.length) const mapping = await create_folders({ source, folders, service_account, root: new_root.id, task_id: record.id }) await copy_files({ files, mapping, service_account, root: new_root.id, task_id: record.id }) db.prepare('update task set status=?, ftime=? where id=?').run('finished', Date.now(), record.id) return { id: new_root.id, task_id: record.id } } else { // ctrl+c 退出 return console.log('退出程序') } } else { const new_root = await get_new_root() const root_mapping = source + ' ' + new_root.id + '\n' const { lastInsertRowid } = db.prepare('insert into task (source, target, status, mapping, ctime) values (?, ?, ?, ?, ?)').run(source, target, 'copying', root_mapping, Date.now()) const arr = await walk_and_save({ fid: source, update, not_teamdrive, service_account }) let files = arr.filter(v => v.mimeType !== FOLDER_TYPE) if (min_size) files = files.filter(v => v.size >= min_size) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) console.log('待複製的目錄數:', folders.length) console.log('待複製的檔案數:', files.length) const mapping = await create_folders({ source, folders, service_account, root: new_root.id, task_id: lastInsertRowid }) await copy_files({ files, mapping, service_account, root: new_root.id, task_id: lastInsertRowid }) db.prepare('update task set status=?, ftime=? where id=?').run('finished', Date.now(), lastInsertRowid) return { id: new_root.id, task_id: lastInsertRowid } } } async function copy_files ({ files, mapping, service_account, root, task_id }) { console.log('\n開始複製文件,總數:', files.length) const limit = pLimit(PARALLEL_LIMIT) let count = 0 const loop = setInterval(() => { const now = dayjs().format('HH:mm:ss') const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}` print_progress(message) }, 1000) return Promise.all(files.map(async file => { const { id, parent } = file const target = mapping[parent] || root const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id)) if (new_file) { count++ db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id) } })).finally(() => clearInterval(loop)) } async function copy_file (id, parent, use_sa, limit, task_id) { let url = `https://www.googleapis.com/drive/v3/files/${id}/copy` let params = { supportsAllDrives: true } url += '?' + params_to_query(params) const config = {} let retry = 0 while (retry < RETRY_LIMIT) { let gtoken if (use_sa) { const temp = await get_sa_token() gtoken = temp.gtoken config.headers = { authorization: 'Bearer ' + temp.access_token } } else { config.headers = await gen_headers() } try { const { data } = await axins.post(url, { parents: [parent] }, config) return data } catch (err) { retry++ handle_error(err) const data = err && err.response && err.response.data const message = data && data.error && data.error.message if (message && message.toLowerCase().includes('file limit')) { if (limit) limit.clearQueue() if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id) throw new Error('您的小組雲端硬碟文件數已超限,停止複製') } if (use_sa && message && message.toLowerCase().includes('rate limit')) { SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken) if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch() console.log('此帳號觸發使用限額,剩餘可用service account帳號數量:', SA_TOKENS.length) } } } if (use_sa && !SA_TOKENS.length) { if (limit) limit.clearQueue() if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id) throw new Error('所有SA帳號流量已用完') } else { console.warn('複製檔案失敗,檔案id: ' + id) } } async function create_folders ({ source, old_mapping, folders, root, task_id, service_account }) { if (!Array.isArray(folders)) throw new Error('folders must be Array:' + folders) const mapping = old_mapping || {} mapping[source] = root if (!folders.length) return mapping const missed_folders = folders.filter(v => !mapping[v.id]) console.log('開始複製資料夾,總數:', missed_folders.length) const limit = pLimit(PARALLEL_LIMIT) let count = 0 let same_levels = folders.filter(v => v.parent === folders[0].parent) const loop = setInterval(() => { const now = dayjs().format('HH:mm:ss') const message = `${now} | 已創建目錄 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}` print_progress(message) }, 1000) while (same_levels.length) { const same_levels_missed = same_levels.filter(v => !mapping[v.id]) await Promise.all(same_levels_missed.map(async v => { try { const { name, id, parent } = v const target = mapping[parent] || root const new_folder = await limit(() => create_folder(name, target, service_account, limit)) count++ mapping[id] = new_folder.id const mapping_record = id + ' ' + new_folder.id + '\n' db.prepare('update task set mapping = mapping || ? where id=?').run(mapping_record, task_id) } catch (e) { if (e.message === FILE_EXCEED_MSG) { clearInterval(loop) throw new Error(FILE_EXCEED_MSG) } console.error('創建目錄出錯:', e.message) } })) // folders = folders.filter(v => !mapping[v.id]) same_levels = [].concat(...same_levels.map(v => folders.filter(vv => vv.parent === v.id))) } clearInterval(loop) return mapping } function find_dupe (arr) { const files = arr.filter(v => v.mimeType !== FOLDER_TYPE) const folders = arr.filter(v => v.mimeType === FOLDER_TYPE) const exists = {} const dupe_files = [] const dupe_folder_keys = {} for (const folder of folders) { const { parent, name } = folder const key = parent + '|' + name if (exists[key]) { dupe_folder_keys[key] = true } else { exists[key] = true } } const dupe_empty_folders = folders.filter(folder => { const { parent, name } = folder const key = parent + '|' + name return dupe_folder_keys[key] }).filter(folder => { const has_child = arr.some(v => v.parent === folder.id) return !has_child }) for (const file of files) { const { md5Checksum, parent, name } = file // 根据文件位置和md5值来判断是否重复 const key = parent + '|' + md5Checksum // + '|' + name if (exists[key]) { dupe_files.push(file) } else { exists[key] = true } } return dupe_files.concat(dupe_empty_folders) } async function confirm_dedupe ({ file_number, folder_number }) { const answer = await prompts({ type: 'select', name: 'value', message: `檢測到同位置下重複文件${file_number}个,重複空目錄${folder_number}個,是否刪除?`, choices: [ { title: 'Yes', description: '確認刪除', value: 'yes' }, { title: 'No', description: '先不刪除', value: 'no' } ], initial: 0 }) return answer.value } // 将文件或文件夹移入回收站,需要 sa 为 content manager 权限及以上 async function trash_file ({ fid, service_account }) { const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true` const headers = await gen_headers(service_account) return axins.patch(url, { trashed: true }, { headers }) } // 直接删除文件或文件夹,不会进入回收站,需要 sa 为 manager 权限 async function rm_file ({ fid, service_account }) { const headers = await gen_headers(service_account) let retry = 0 const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true` while (retry < RETRY_LIMIT) { try { return await axins.delete(url, { headers }) } catch (err) { retry++ handle_error(err) console.log('刪除重試中,重試次數', retry) } } } async function dedupe ({ fid, update, service_account }) { let arr if (!update) { const info = get_all_by_fid(fid) if (info) { console.log('找到本地快取資料,快取時間:', dayjs(info.mtime).format('YYYY-MM-DD HH:mm:ss')) arr = info } } arr = arr || await walk_and_save({ fid, update, service_account }) const dupes = find_dupe(arr) const folder_number = dupes.filter(v => v.mimeType === FOLDER_TYPE).length const file_number = dupes.length - folder_number const choice = await confirm_dedupe({ file_number, folder_number }) if (choice === 'no') { return console.log('退出程序') } else if (!choice) { return // ctrl+c } const limit = pLimit(PARALLEL_LIMIT) let folder_count = 0 let file_count = 0 await Promise.all(dupes.map(async v => { try { await limit(() => trash_file({ fid: v.id, service_account })) if (v.mimeType === FOLDER_TYPE) { console.log('成功刪除資料夾', v.name) folder_count++ } else { console.log('成功刪除檔案', v.name) file_count++ } } catch (e) { console.log('刪除失敗', e.message) } })) return { file_count, folder_count } } function handle_error (err) { const data = err && err.response && err.response.data if (data) { console.error(JSON.stringify(data)) } else { if (!err.message.includes('timeout')) console.error(err.message) } } function print_progress (msg) { if (process.stdout.cursorTo) { process.stdout.cursorTo(0) process.stdout.write(msg + ' ') } else { console.log(msg) } } module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id }