Merge 3fe80f3251
into 8ae95bbb2d
This commit is contained in:
commit
eb09795713
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,4 +1,5 @@
|
||||
node_modules/
|
||||
gdurl.sqlite
|
||||
gdurl.sqlite*
|
||||
config.js
|
||||
sa/*.json
|
||||
backup/*.sqlite
|
14
backup-db.js
Normal file
14
backup-db.js
Normal file
@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const path = require('path')
|
||||
const {db} = require('./db')
|
||||
|
||||
const filepath = path.join(__dirname, 'backup', `${Date.now()}.sqlite`)
|
||||
|
||||
db.backup(filepath)
|
||||
.then(() => {
|
||||
console.log(filepath)
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log('backup failed:', err)
|
||||
})
|
0
backup/.keep
Normal file
0
backup/.keep
Normal file
28
bookmark.js
Normal file
28
bookmark.js
Normal file
@ -0,0 +1,28 @@
|
||||
const fs = require('fs')
|
||||
const {db} = require('./db')
|
||||
|
||||
const action = process.argv[2] || 'export'
|
||||
const filepath = process.argv[3] || 'bookmarks.json'
|
||||
|
||||
if (action === 'export') {
|
||||
const bookmarks = db.prepare('select * from bookmark').all()
|
||||
fs.writeFileSync(filepath, JSON.stringify(bookmarks))
|
||||
console.log('bookmarks exported', filepath)
|
||||
} else if (action === 'import') {
|
||||
let bookmarks = fs.readFileSync(filepath, 'utf8')
|
||||
bookmarks = JSON.parse(bookmarks)
|
||||
bookmarks.forEach(v => {
|
||||
const {alias, target} = v
|
||||
const exist = db.prepare('select alias from bookmark where alias=?').get(alias)
|
||||
if (exist) {
|
||||
db.prepare('update bookmark set target=? where alias=?').run(target, alias)
|
||||
} else {
|
||||
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
|
||||
}
|
||||
})
|
||||
console.log('bookmarks imported', bookmarks)
|
||||
} else {
|
||||
console.log('[help info]')
|
||||
console.log('export: node bookmark.js export bm.json')
|
||||
console.log('import: node bookmark.js import bm.json')
|
||||
}
|
8
clear-db.js
Normal file
8
clear-db.js
Normal file
@ -0,0 +1,8 @@
|
||||
const { db } = require('./db')
|
||||
|
||||
const record = db.prepare('select count(*) as c from gd').get()
|
||||
db.prepare('delete from gd').run()
|
||||
console.log('已刪除', record.c, '項資料')
|
||||
|
||||
db.exec('vacuum')
|
||||
db.close()
|
25
config.js
25
config.js
@ -1,27 +1,24 @@
|
||||
// 单次请求多少毫秒未响应以后超时(基准值,若连续超时则下次调整为上次的2倍)
|
||||
// 單次請求多少毫秒無回應以後超時(基準值,若連續超時則下次調整為上次的2倍)
|
||||
const TIMEOUT_BASE = 7000
|
||||
// 最大超时设置,比如某次请求,第一次7s超时,第二次14s,第三次28s,第四次56s,第五次不是112s而是60s,后续同理
|
||||
// 最大超時設置,比如某次請求,第一次7s超時,第二次14s,第三次28s,第四次56s,第五次不是112s而是60s,後續同理
|
||||
const TIMEOUT_MAX = 60000
|
||||
|
||||
const LOG_DELAY = 5000 // 日志输出时间间隔,单位毫秒
|
||||
const PAGE_SIZE = 1000 // 每次网络请求读取目录下的文件数,数值越大,越有可能超时,不得超过1000
|
||||
const LOG_DELAY = 5000 // 日志輸出時間間隔,單位毫秒
|
||||
const PAGE_SIZE = 1000 // 每次網絡請求讀取目錄下的文件數,數值越大,越有可能超時,不得超過1000
|
||||
|
||||
const RETRY_LIMIT = 7 // 如果某次请求失败,允许其重试的最大次数
|
||||
const PARALLEL_LIMIT = 20 // 网络请求的并行数量,可根据网络环境调整
|
||||
const RETRY_LIMIT = 7 // 如果某次請求失敗,允許其重試的最大次數
|
||||
const PARALLEL_LIMIT = 20 // 網絡請求的並行數量,可根據網絡環境調整(即多線程之線程數量)
|
||||
|
||||
const DEFAULT_TARGET = '' // 必填,拷贝默认目的地ID,如果不指定target,则会复制到此处,建议填写团队盘ID
|
||||
const DEFAULT_TARGET = '' // 必填,copy時預設的dstID,建議填寫小組雲端硬碟ID
|
||||
|
||||
const AUTH = { // 如果您拥有service account的json授权文件,可将其拷贝至 sa 目录中以代替 client_id/secret/refrest_token
|
||||
const AUTH = { // 如果您擁有SA的json授權文件,可將其拷貝至 sa 目錄中以代替 client_id/secret/refrest_token 這裡建議使用自己的client_id, 具體參考說明文件#個人帳號配置
|
||||
client_id: 'your_client_id',
|
||||
client_secret: 'your_client_secret',
|
||||
refresh_token: 'your_refrest_token',
|
||||
expires: 0, // 可以留空
|
||||
access_token: '', // 可以留空
|
||||
tg_token: 'bot_token', // 你的 telegram robot 的 token,获取方法参见 https://core.telegram.org/bots#6-botfather
|
||||
tg_whitelist: ['your_tg_username'] // 你的tg username(t.me/username),bot只会执行这个列表里的用户所发送的指令
|
||||
tg_token: 'bot_token', // 你的 telegram robot 的 token,獲取方法參見 https://core.telegram.org/bots#6-botfather
|
||||
tg_whitelist: ['your_tg_username'] // 你的tg username(t.me/username),bot只會執行這個列表中的用戶所發送的指令
|
||||
}
|
||||
|
||||
//-------------------MOD-------------------
|
||||
const SA_PATH = '../sa' //sa路徑配置, 給定絕對路徑或是以src為當前路徑給定相對路徑, 預設為'../sa'
|
||||
const BUTTON_LEVEL = 1 //預設為1, 填入大於2皆視為2
|
||||
module.exports = { AUTH, PARALLEL_LIMIT, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET, SA_PATH, BUTTON_LEVEL }
|
||||
module.exports = { AUTH, PARALLEL_LIMIT, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET }
|
||||
|
5
config_mod.js
Normal file
5
config_mod.js
Normal file
@ -0,0 +1,5 @@
|
||||
//-------------------MOD-------------------
|
||||
const SA_PATH = '../sa' //sa路徑配置, 給定絕對路徑或是以src為當前路徑給定相對路徑, 預設為'../sa'
|
||||
const BUTTON_LEVEL = 1 //預設為1, 填入大於2皆視為2
|
||||
|
||||
module.exports = { SA_PATH, BUTTON_LEVEL }
|
24
copy
24
copy
@ -5,19 +5,21 @@ const bytes = require('bytes')
|
||||
const { argv } = require('yargs')
|
||||
.usage('用法: ./$0 <source id> <target id> [options]\ntarget id可选,不填则使用config.js里的DEFAULT_TARGET')
|
||||
.alias('u', 'update')
|
||||
.describe('u', '不使用本地缓存,强制从线上获取源文件夹信息')
|
||||
.describe('u', '不使用本地快取,則無視快取記錄強制從線上獲取源資料夾資訊')
|
||||
.alias('y', 'yes')
|
||||
.describe('yes', '如果發現拷貝紀錄,直接繼續上次的進度')
|
||||
.alias('f', 'file')
|
||||
.describe('f', '复制单个文件')
|
||||
.describe('f', '複製單一文件')
|
||||
.alias('n', 'name')
|
||||
.describe('n', '给目标文件夹重命名,不填则保留原始目录名')
|
||||
.describe('n', '給目標資料夾重新命名,不填則保留原始目錄名')
|
||||
.alias('N', 'not_teamdrive')
|
||||
.describe('N', '如果不是团队盘链接,可以加上此参数以提高接口查询效率,降低延迟')
|
||||
.describe('N', '如果不是小組雲端硬碟連結,可以加上此参數以提高接口查詢效率,降低延遲')
|
||||
.alias('s', 'size')
|
||||
.describe('s', '不填默认拷贝全部文件,如果设置了这个值,则过滤掉小于这个size的文件,必须以b结尾,比如10mb')
|
||||
.describe('s', '不填則預設拷貝全部文件,如果設置了这個值,則過濾掉小於这個大小的檔案,必須以b為結尾,比如10mb')
|
||||
.alias('S', 'service_account')
|
||||
.describe('S', '指定使用service account进行操作,前提是必须在 ./sa 目录下放置json授权文件,请确保sa帐号拥有操作权限。')
|
||||
.describe('S', '指定使用service account進行操作,前提是必須在 ./sa 目錄下放置json授權文件,請確認sa帳號擁有相關操作權限。')
|
||||
.alias('D', 'dncnr')
|
||||
.describe('D', 'do not create new root, 不在目的地创建同名文件夹,直接将源文件夹中的文件原样复制到目的文件夹中')
|
||||
.describe('D', 'do not create new root, 不在目的地創建同名資料夾,直接將來源資料夾中的文件原樣複製到目的資料夾中')
|
||||
.help('h')
|
||||
.alias('h', 'help')
|
||||
|
||||
@ -33,19 +35,19 @@ if (validate_fid(source)) {
|
||||
if (!validate_fid(target)) throw new Error('target id 格式不正确')
|
||||
return copy_file(source, target, service_account).then(r => {
|
||||
const link = 'https://drive.google.com/drive/folders/' + target
|
||||
console.log('任务完成,文件所在位置:\n', link)
|
||||
console.log('任務完成,文件所在位置:\n', link)
|
||||
}).catch(console.error)
|
||||
}
|
||||
let min_size
|
||||
if (size) {
|
||||
console.log(`不复制大小低于 ${size} 的文件`)
|
||||
console.log(`不複製大小低於 ${size} 的文件`)
|
||||
min_size = bytes.parse(size)
|
||||
}
|
||||
copy({ source, target, name, min_size, update, not_teamdrive, service_account, dncnr }).then(folder => {
|
||||
if (!folder) return
|
||||
const link = 'https://drive.google.com/drive/folders/' + folder.id
|
||||
console.log('\n任务完成,新文件夹链接:\n', link)
|
||||
console.log('\n任務完成,新資料夾連結:\n', link)
|
||||
})
|
||||
} else {
|
||||
console.warn('目录ID缺失或格式错误')
|
||||
console.warn('無目錄ID或格式錯誤')
|
||||
}
|
||||
|
24
count
24
count
@ -1,23 +1,23 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { argv } = require('yargs')
|
||||
.usage('用法: ./$0 <目录ID> [options]')
|
||||
.example('./$0 1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75', '获取 https://drive.google.com/drive/folders/1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75 内包含的的所有文件的统计信息')
|
||||
.example('./$0 root -s size -t html -o out.html', '获取个人盘根目录统计信息,结果以HTML表格输出,根据总大小逆序排列,保存到本目录下的out.html文件中(不存在则新建,存在则覆盖)')
|
||||
.example('./$0 root -s name -t json -o out.json', '获取个人盘根目录统计信息,结果以JSON格式输出,根据文件扩展名排序,保存到本目录下的out.json文件中')
|
||||
.example('./$0 root -t all -o all.json', '获取个人盘根目录统计信息,将所有文件信息(包括文件夹)以JSON格式输出,保存到本目录下的all.json文件中')
|
||||
.usage('用法: ./$0 <目錄ID> [options]')
|
||||
.example('./$0 1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75', '獲取 https://drive.google.com/drive/folders/1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75 内包含的所有文件的統計資訊')
|
||||
.example('./$0 root -s size -t html -o out.html', '獲取個人空間根目錄統計資訊,結果以HTML表格輸出,根據總大小降冪排列,保存到本目錄下的out.html文件中(不存在則新建,存在的覆蓋)')
|
||||
.example('./$0 root -s name -t json -o out.json', '獲取個人空間根目錄統計資訊,結果以JSON格式輸出,根據文件扩展名排序,保存到本目录下的out.json文件中')
|
||||
.example('./$0 root -t all -o all.json', '獲取個人空間根目錄統計資訊,將所有文件資訊(包括資料夾)以JSON格式輸出,保存到本目錄下的all.json文件中')
|
||||
.alias('u', 'update')
|
||||
.describe('u', '强制从线上获取信息(无视是否存在本地缓存)')
|
||||
.describe('u', '強制從線上獲取資訊(不論是否存在本地快取)')
|
||||
.alias('N', 'not_teamdrive')
|
||||
.describe('N', '如果不是团队盘链接,可以加上此参数以提高接口查询效率,降低延迟。如果要统计的是个人盘且./sa目录下的service account没有相关权限,请确保加上此参数以使用个人的auth信息进行查询')
|
||||
.describe('N', '如果不是小組雲端硬碟連結,可以加上此参數以提高接口查詢效率,降低延遲。如果要統計的是帳號下之個人空間,且./sa目錄下的service account沒有相關權限,請加上此參數以使用個人的auth身份進行查詢')
|
||||
.alias('S', 'service_account')
|
||||
.describe('S', '指定使用service account进行统计,前提是必须在sa目录下放置SA json文件')
|
||||
.describe('S', '指定使用service account進行統計,前提是必須在sa路徑下放置SA json文件')
|
||||
.alias('s', 'sort')
|
||||
.describe('s', '统计结果排序方法,可选值 name 或 size,不填则默认根据文件数量逆序排列')
|
||||
.describe('s', '統計結果排序方法,可選值如下: name 或 size,不填則預設根據文件數量降冪排列')
|
||||
.alias('t', 'type')
|
||||
.describe('t', '统计结果输出类型,可选值 html/json/all,all表示输出所有文件json数据,最好搭配 -o 使用。不填则默认输出命令行表格')
|
||||
.describe('t', '統計結果輸出類型,可選值如下: html/tree/json/all,all表示輸出所有文件json數據,最好與 -o 一起使用。不填則預設輸出命令列表格')
|
||||
.alias('o', 'output')
|
||||
.describe('o', '统计结果输出文件,适合搭配 -t 使用')
|
||||
.describe('o', '統計結果輸出文件,適合與 -t 一起使用')
|
||||
.help('h')
|
||||
.alias('h', 'help')
|
||||
|
||||
@ -27,5 +27,5 @@ if (validate_fid(fid)) {
|
||||
const { update, sort, type, output, not_teamdrive, service_account } = argv
|
||||
count({ fid, update, sort, type, output, not_teamdrive, service_account }).catch(console.error)
|
||||
} else {
|
||||
console.warn('目录ID缺失或格式错误')
|
||||
console.warn('無目錄ID或格式錯誤')
|
||||
}
|
||||
|
12
dedupe
12
dedupe
@ -1,11 +1,13 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { argv } = require('yargs')
|
||||
.usage('用法: ./$0 <source id> <target id> [options]')
|
||||
.usage('用法: ./$0 <folder-id> [options]')
|
||||
.alias('y', 'yes')
|
||||
.describe('yes', '如果發現重複項目,直接刪除')
|
||||
.alias('u', 'update')
|
||||
.describe('u', '不使用本地缓存,强制从线上获取源文件夹信息')
|
||||
.describe('u', '不使用本地快取,則無視快取記錄強制從線上獲取源資料夾資訊')
|
||||
.alias('S', 'service_account')
|
||||
.describe('S', '使用service account进行操作,前提是必须在 ./sa 目录下放置sa授权json文件')
|
||||
.describe('S', '使用service account進行操作,前提是必須在 ./sa 目錄下存放sa授權json文件')
|
||||
.help('h')
|
||||
.alias('h', 'help')
|
||||
|
||||
@ -17,8 +19,8 @@ if (validate_fid(fid)) {
|
||||
dedupe({ fid, update, service_account }).then(info => {
|
||||
if (!info) return
|
||||
const { file_count, folder_count } = info
|
||||
console.log('任务完成,共删除文件数:', file_count, '目录数:', folder_count)
|
||||
console.log('任務完成,共刪除檔案數:', file_count, '目錄數:', folder_count)
|
||||
})
|
||||
} else {
|
||||
console.warn('目录ID缺失或格式错误')
|
||||
console.warn('無目錄ID或格式錯誤')
|
||||
}
|
||||
|
245
gdutilsinstall_cf.sh
Normal file
245
gdutilsinstall_cf.sh
Normal file
@ -0,0 +1,245 @@
|
||||
#!/bin/bash
|
||||
echo
|
||||
echo -e "\033[1;32m===== <<gdutils项目一件部署脚本要求及说明>> =====\033[0m"
|
||||
echo -e "\033[1;32m---------------[ v2.1 by oneking ]---------------\033[0m"
|
||||
echo -e "\033[32m 01.\033[0m 本脚本是针对TG大神@viegg的gdutils项目一键部署脚本;"
|
||||
echo -e "\033[32m 02.\033[0m 脚本包括“TD盘VPS上查询转存部署”和“Telegram机器人部署”两部分"
|
||||
echo -e "\033[32m 03.\033[0m 本脚本适应CentOS/Debian/Ubuntu三种操作系统,自动识别、自动选择对应分支一键安装部署"
|
||||
echo -e "\033[32m 04.\033[0m 三步即可完成部署:上传脚本到VPS → 设置脚本执行权限 → 运行"
|
||||
echo -e "\033[32m 05.\033[0m 准备工作一:在TG上注册好机器人取得并记录下该机器人TOKEN"
|
||||
echo -e "\033[32m 06.\033[0m 准备工作二:拥有一个域名绑定到cloudflare解析到该机器人所在服务器IP"
|
||||
echo -e "\033[32m 07.\033[0m 准备工作三:向机器人@userinfobot获取个人TG账号ID并记录"
|
||||
echo -e "\033[32m 08.\033[0m 准备工作四:注册好一个Google team drive加入sa并记录下该盘ID"
|
||||
echo -e "\033[32m 09.\033[0m 经测试可用完美安装系统:Centos 7/8 debian 9/10 ubuntu 16.04/18.04/19.10/20.04"
|
||||
echo -e "\033[32m 10.\033[0m 部署过程中有任何问题请把“错误截图”“部署VPS系统名称版本”信息发给TG:onekings 或 vitaminor@gmail.com"
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
read -s -n1 -p "★★★ 如已做好以上[5/6/7/8]准备或不需要安装Telegram机器人请按任意键开始部署,如未做好准备请按“Ctrl+c”终止脚本 ★★★"
|
||||
echo
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
|
||||
# 识别操作系统
|
||||
aNAME="$(uname -a)"
|
||||
bNAME="$(cat /proc/version)"
|
||||
cNAME="$(lsb_release -a)"
|
||||
if [ -f "/etc/redhat-release" ]; then
|
||||
if [[ $(cat /etc/redhat-release) =~ "CentOS" ]]; then
|
||||
os="CentOS"
|
||||
fi
|
||||
elif [ "$aNAME"=~"Debian" -o "$bNAME"=~"Debian" -o "$cNAME"=~"Debian" ]; then
|
||||
os="Debian"
|
||||
elif [ "$aNAME"=~"Ubuntu" -o "$bNAME"=~"Ubuntu" -o "$cNAME"=~"Ubuntu" ]; then
|
||||
os="Debian"
|
||||
elif [ "$aNAME"=~"CentOS" -o "$bNAME"=~"CentOS" -o "$cNAME"=~"CentOS" ]; then
|
||||
os="CentOS"
|
||||
elif [ "$aNAME"=~"Darwin" -o "$bNAME"=~"Darwin" -o "$cNAME"=~"Darwin" ]; then
|
||||
os="mac"
|
||||
else
|
||||
os="$bNAME"
|
||||
fi
|
||||
|
||||
# 需要安装的软件工具及依赖
|
||||
insofts=(epel-release update upgrade wget curl git unzip zip python3-distutils python3 python3-pip)
|
||||
|
||||
#根据操作系统设置变量
|
||||
if [[ "$os" = "Debian" ]]; then
|
||||
cmd_install="apt-get" #安装命令
|
||||
cmd_install_rely="build-essential" #c++编译环境
|
||||
nodejs_curl="https://deb.nodesource.com/setup_10.x" #nodejs下载链接
|
||||
cmd_install_rpm_build="" #安装rpm-build
|
||||
nginx_conf="/etc/nginx/sites-enabled/" #nginx配置文件存放路径
|
||||
rm_nginx_default="rm -f /etc/nginx/sites-enabled/default" #删除default
|
||||
echo
|
||||
echo -e "\033[1;32m★★★★★ 您的操作系统为Debian,即将为你开始部署gdutils项目 ★★★★★\033[0m"
|
||||
elif [[ "$os" = "Ubuntu" ]]; then
|
||||
cmd_install="sudo apt-get"
|
||||
cmd_install_rely="build-essential"
|
||||
nodejs_curl="https://deb.nodesource.com/setup_10.x"
|
||||
cmd_install_rpm_build=""
|
||||
nginx_conf="/etc/nginx/sites-enabled/"
|
||||
rm_nginx_default="rm -f /etc/nginx/sites-enabled/default"
|
||||
echo
|
||||
echo -e "\033[1;32m★★★★★ 您的操作系统为Ubuntu,即将为你开始部署gdutils项目 ★★★★★\033[0m"
|
||||
elif [[ "$os" = "CentOS" ]]; then
|
||||
cmd_install="yum"
|
||||
cmd_install_rely="gcc-c++ make"
|
||||
nodejs_curl="https://rpm.nodesource.com/setup_10.x"
|
||||
cmd_install_rpm_build="yum install rpm-build -y"
|
||||
nginx_conf="/etc/nginx/conf.d/"
|
||||
rm_nginx_default=""
|
||||
echo
|
||||
echo -e "\033[1;32m★★★★★ 您的操作系统为Centos,即将为你开始部署gdutils项目 ★★★★★\033[0m"
|
||||
elif [[ "$os" = "mac" ]]; then
|
||||
echo
|
||||
echo -e "\033[1;32m★★★★★ 您的操作系统为MacOS,请在图形界面手动安装 ★★★★★\033[0m"
|
||||
exit
|
||||
echo
|
||||
echo
|
||||
else
|
||||
echo
|
||||
echo -e "\033[1;32m unknow os $OS, exit! \033[0m"
|
||||
exit
|
||||
echo
|
||||
echo
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "\033[1;32m===== <<升级系统/更新软件/安装工具/安装依赖>> =====\033[0m"
|
||||
echo
|
||||
|
||||
#安装which和sudo
|
||||
if [[ "$(which which)" == "" ]]; then
|
||||
echo -e "\033[1;32m“which”开始安装......\033[0m"
|
||||
$cmd_install install which -y
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
elif [[ "$(which sudo)" == "" ]]; then
|
||||
echo -e "\033[1;32m“sudo”开始安装......\033[0m"
|
||||
$cmd_install install sudo -y
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
fi
|
||||
|
||||
#安装工具和依赖
|
||||
for ((aloop = 0; aloop < ${#insofts[@]}; aloop++)); do
|
||||
if [ ${insofts[$aloop]} = "update" -o ${insofts[$aloop]} = "upgrade" ]; then
|
||||
echo -e "\033[1;32m“${insofts[$aloop]}”开始安装......\033[0m"
|
||||
$cmd_install ${insofts[$aloop]} -y
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
else
|
||||
echo -e "\033[1;32m“${insofts[$aloop]}”开始安装......\033[0m"
|
||||
$cmd_install install ${insofts[$aloop]} -y
|
||||
echo -e "\033[1;32m------------------------------------------------\033[0m"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo -e "\033[1;32m===== <<安装gdutils依赖-nodejs和npm/安装配置gdutils>> =====\033[0m"
|
||||
echo
|
||||
$cmd_install install $cmd_install_rely -y
|
||||
curl -sL $nodejs_curl | bash -
|
||||
$cmd_install install nodejs -y
|
||||
$cmd_install_rpm_build
|
||||
git clone https://github.com/liaojack8/gd-utils-cht && cd gd-utils-cht
|
||||
npm config set unsafe-perm=true
|
||||
npm i
|
||||
|
||||
echo
|
||||
echo -e "\033[1;32m★★★ 恭喜您!gdutils统计转存系统已经正确安装完成,请上传sa到“./gd-utils-cht/sa/”目录下完成最后的配置 ★★★\033[0m"
|
||||
echo
|
||||
|
||||
#################################################################################################
|
||||
|
||||
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
|
||||
read -s -n1 -p "★★★ 下面将部署Telegram机器人,请确保准备所需条件已准备好,按任意键开始部署机器人;如未做好准备请按“Ctrl+c”终止部署机器人 ★★★"
|
||||
echo
|
||||
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
|
||||
|
||||
echo
|
||||
echo -e "\033[1;32m ===== <<开始部署gdutils查询转存TG机器人>> ===== \033[0m"
|
||||
echo
|
||||
|
||||
#输入“机器人token/TG账号ID/域名/转存目的盘ID”
|
||||
read -p """请输入机器人token并回车
|
||||
Your Bot Token =>:""" YOUR_BOT_TOKEN
|
||||
#判断token是否输入正确
|
||||
while [[ "${#YOUR_BOT_TOKEN}" != 46 ]]; do
|
||||
echo -e "\033[1;32m★★★ 机器人TOKEN输入不正确,请重新输入或按“Ctrl+C”结束安装! ★★★\033[0m"
|
||||
read -p """请输入机器人token并回车
|
||||
Your Bot Token =>:""" YOUR_BOT_TOKEN
|
||||
done
|
||||
|
||||
read -p """请输入你的域名(在cloudflare上解析到你机器人所在VPS的域名,格式:bot.abc.com)并回车
|
||||
Your Domain Name =>:""" YOUR_DOMAIN_NAME
|
||||
#判断域名是否正确
|
||||
while [[ "$YOUR_DOMAIN_NAME" =~ "http" ]]; do
|
||||
echo -e "\033[1;32m★★★ “Your Domain Name”输入错误,应该输入你在cloudflare上解析的域名且不包含“http”,请重新输入或按“Ctrl+C”结束安装! ★★★\033[0m"
|
||||
read -p """请输入你的域名(在cloudflare上解析到你机器人所在VPS的域名,格式:bot.abc.com)并回车
|
||||
Your Domain Name =>:""" YOUR_DOMAIN_NAME
|
||||
done
|
||||
|
||||
read -p """请输入使用机器人的telegram账号ID(获取ID机器人@userinfobot)并回车
|
||||
Your Telegram ID =>:""" YOUR_TELEGRAM_ID
|
||||
#判断telegram ID是否正确(通过判断是不是纯数字)
|
||||
until [[ $YOUR_TELEGRAM_ID =~ ^-?[0-9]+$ ]]; do
|
||||
echo -e "\033[1;32m★★★ 您的TG账号ID输入不正确,请重新输入或按“Ctrl+C”结束安装! ★★★\033[0m"
|
||||
read -p """请输入使用机器人的telegram账号ID(获取ID机器人@userinfobot)并回车
|
||||
Your Telegram ID =>:""" YOUR_TELEGRAM_ID
|
||||
done
|
||||
|
||||
read -p """请输入转存默认目的地团队盘ID(不指定转存目的地默认改地址,脚本强制要求输入团队盘ID)并回车
|
||||
Your Google Team Drive ID =>:""" YOUR_GOOGLE_TEAM_DRIVE_ID
|
||||
#判断google team drive ID是否正确(团队盘ID长度19位)
|
||||
while [[ "${#YOUR_GOOGLE_TEAM_DRIVE_ID}" != 19 ]]; do
|
||||
echo -e "\033[1;32m★★★ 您的Google team drive ID输入不正确,请重新输入或按“Ctrl+C”结束安装! ★★★\033[0m"
|
||||
read -p """请输入转存默认目的地ID(不指定转存目的地默认该地址,脚本强制要求输入团队盘ID)并回车
|
||||
Your Google Team Drive ID =>:""" YOUR_GOOGLE_TEAM_DRIVE_ID
|
||||
done
|
||||
|
||||
cd ~ &&
|
||||
sed -i "s/bot_token/$YOUR_BOT_TOKEN/g" ./gd-utils-cht/config.js &&
|
||||
sed -i "s/your_tg_username/$YOUR_TELEGRAM_ID/g" ./gd-utils-cht/config.js &&
|
||||
sed -i "s/DEFAULT_TARGET = ''/DEFAULT_TARGET = '$YOUR_GOOGLE_TEAM_DRIVE_ID'/g" ./gd-utils-cht/config.js
|
||||
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
|
||||
|
||||
echo -e "\033[1;32m“进程守护程序pm2”开始安装......\033[0m"
|
||||
cd /root/gd-utils-cht &&
|
||||
npm i pm2 -g && pm2 l
|
||||
echo -e "\033[1;32m启动守护进程......\033[0m"
|
||||
pm2 start server.js --node-args="--max-old-space-size=4096"
|
||||
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
|
||||
|
||||
echo -e "\033[1;32m“nginx”开始安装......\033[0m"
|
||||
cd ~ &&
|
||||
$cmd_install install nginx -y
|
||||
echo
|
||||
echo -e "\033[1;32m===== <<配置nginx服务>> ===== \033[0m"
|
||||
echo
|
||||
echo -e "\033[1;32m“nginx”起一个web服务......\033[0m"
|
||||
|
||||
cd $nginx_conf
|
||||
echo "server {
|
||||
listen 80;
|
||||
server_name $YOUR_DOMAIN_NAME;
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:23333/;
|
||||
}
|
||||
}" >${nginx_conf}gdutilsbot.conf &&
|
||||
$rm_nginx_default
|
||||
|
||||
ls &&
|
||||
nginx -t &&
|
||||
nginx -c /etc/nginx/nginx.conf &&
|
||||
nginx -s reload &&
|
||||
netstat -tulpen
|
||||
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
|
||||
|
||||
echo -e "\033[1;32m“检查网站是否部署成功”......\033[0m"
|
||||
curl $YOUR_DOMAIN_NAME/api/gdurl/count\?fid=124pjM5LggSuwI1n40bcD5tQ13wS0M6wg
|
||||
echo
|
||||
echo -e "\033[1;32m设置Webhook服务......\033[0m"
|
||||
print_webhook=$(curl -F "url=https://$YOUR_DOMAIN_NAME/api/gdurl/tgbot" "https://api.telegram.org/bot$YOUR_BOT_TOKEN/setWebhook")
|
||||
echo
|
||||
|
||||
# 判断反向代理是否部署成功
|
||||
if [[ $print_webhook =~ "true" ]]; then
|
||||
echo -e "\033[1;32m★★★ 恭喜你!GoogleDrive查询转存机器人部署成功,请回到TG界面给bot发送个“/help”获取使用帮助 ★★★\033[0m"
|
||||
else
|
||||
echo -e "\033[32m★★★ 很遗憾!机器人设置失败,请返回检查网站是否部署成功,并重复本安装过程 ★★★\033[0m", exit!
|
||||
fi
|
||||
nginx -t && nginx -s reload
|
||||
echo
|
||||
echo
|
||||
|
||||
cd ~
|
||||
rm -f gdutilsinstall.sh
|
||||
|
||||
###########################gdutils功能建议##################################
|
||||
# 本部分是对gdutils项目的建议,因为我主要用的是查询功能所以以下建议只涉及查询功能
|
||||
# 1-把以下参数放入配置文件设置:sa存放路径
|
||||
# 2-改sa“随机”使用为“顺序”分组使用;
|
||||
# 3-增加输出模式,可以用命令行后带参数选择,具体模式建议:
|
||||
# ①按一级或者二级文件夹显示数量大小
|
||||
# ②可以一次性统计多个磁盘并且输出单个磁盘文件数和大小以及几个磁盘总和
|
||||
# ③获取id对应的文件夹名或者磁盘明保存数据库,给个命令能够查询历史记录汇总或者指定汇总
|
||||
# 4-查询过程中输出方式不要每次都输出一次,可以固定+数字变化
|
||||
# 5-命令参数可加在ID前或后,如果非要固定一种的话就加在ID之前
|
||||
# 6-命令行也改为默认sa模式
|
||||
############################################################################
|
31
readme.md
31
readme.md
@ -3,11 +3,34 @@
|
||||
> 不只是最快的 google drive 拷貝工具 [與其他工具的對比](./compare.md)
|
||||
|
||||
> 我的readme可能不夠完全, 主要寫上我更新、修改的內容, 具體說明還是看[這邊](https://github.com/iwestlin/gd-utils)和[這邊](https://github.com/vitaminx/gd-utils)吧
|
||||
## 從其他專案轉移至本繁中專案
|
||||
```sh
|
||||
pm2 delete 0
|
||||
mv ./gd-utils/sa ./gd-utils/config.js ./gd-utils/gdurl.sqlite ./
|
||||
rm -rf gd-utils
|
||||
git clone https://github.com/liaojack8/gd-utils-cht && cd gd-utils-cht
|
||||
npm install
|
||||
rm -rf sa config.js gdurl.sqlite
|
||||
cd ..
|
||||
mv sa config.js gdurl.sqlite ./gd-utils-cht/
|
||||
pm2 start ./gd-utils-cht/server.js
|
||||
sudo pm2 save
|
||||
```
|
||||
- Demo Video: ~~[https://drive.google.com/file/d/1CltOaBDa4FVQ6doBP3S84MFPpbs2tv88](https://drive.google.com/file/d/1CltOaBDa4FVQ6doBP3S84MFPpbs2tv88)~~ (My account has been blocked)
|
||||
## 更新紀錄
|
||||
具體功能參考[iwestlin-changelog](https://github.com/iwestlin/gd-utils/blob/master/changelog.md),前期工作基本做完,之後大概就是搬運了,可能考慮做一下i18n
|
||||
### 2020.07.30
|
||||
- 同步原作者之更新 (清除按鈕、aria2.js等)
|
||||
- 依舊保留了config_mod.js的修改項目, button-level的部分也套用清除按鈕的規則
|
||||
### 2020.07.16
|
||||
- 新增了從其他版本轉移到本專案的方式及教學
|
||||
- 改用config_mod.js, 可自訂按鈕顯示的個數(每列), 可設定為1或2, 或自訂sa文件路徑(此文件不修改也可以正常使用bot)
|
||||
### 2020.07.15
|
||||
- 參照原作者更新, 加入單檔複製、tree列表的功能
|
||||
### 2020.07.07
|
||||
- 參照原作者@iwestlin更新tg.js及gd.js
|
||||
- 整體繁體化, 介面部分
|
||||
- 新增用戶可以在config.js自訂按鈕顯示的個數(每列), 可設定為1或2
|
||||
- ~~新增用戶可以在config.js自訂按鈕顯示的個數(每列), 可設定為1或2~~
|
||||
### 2020.07.06
|
||||
- 部分繁體中文化
|
||||
- 執行/task命令時, 會回傳完成度百分比
|
||||
@ -30,15 +53,15 @@
|
||||
- gdutils項目一鍵部署腳本(包括“查詢轉存”和“TG機器人”兩部分)
|
||||
```
|
||||
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilsinstall.sh)"
|
||||
```
|
||||
```
|
||||
- gdutils項目一鍵部署腳本之“轉存查詢部分”
|
||||
```
|
||||
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilscsinstall.sh)"
|
||||
```
|
||||
```
|
||||
- gdutils項目一鍵部署腳本之“TG機器人部分”
|
||||
```
|
||||
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilsbotinstall.sh)"
|
||||
```
|
||||
```
|
||||
- 安裝過程中需要輸入一下四個參數:
|
||||
- 機器人TOKEN:這個在Telegram裡面找“@BotFather”註冊即可獲得
|
||||
- Telegram用戶ID:在Telegram裡面向機器人@userinfobot发送消息即可獲得
|
||||
|
172
src/gd.js
172
src/gd.js
@ -4,28 +4,57 @@ const dayjs = require('dayjs')
|
||||
const prompts = require('prompts')
|
||||
const pLimit = require('p-limit')
|
||||
const axios = require('@viegg/axios')
|
||||
const HttpsProxyAgent = require('https-proxy-agent')
|
||||
const { GoogleToken } = require('gtoken')
|
||||
const handle_exit = require('signal-exit')
|
||||
const { argv } = require('yargs')
|
||||
|
||||
const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET, SA_PATH } = require('../config')
|
||||
let { PARALLEL_LIMIT } = require('../config')
|
||||
PARALLEL_LIMIT = argv.l || argv.limit || PARALLEL_LIMIT
|
||||
|
||||
const { AUTH, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET } = require('../config')
|
||||
const { SA_PATH } = require('../config_mod')
|
||||
const { db } = require('../db')
|
||||
const { make_table, make_tg_table, make_html, summary } = require('./summary')
|
||||
const { gen_tree_html } = require('./tree')
|
||||
|
||||
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製'
|
||||
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製,請將未完成的資料夾移到另一個小組雲端硬碟中,再執行一遍複製指令即可繼斷點續傳'
|
||||
const FOLDER_TYPE = 'application/vnd.google-apps.folder'
|
||||
const { https_proxy } = process.env
|
||||
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
|
||||
const sleep = ms => new Promise((resolve, reject) => setTimeout(resolve, ms))
|
||||
|
||||
const { https_proxy, http_proxy, all_proxy } = process.env
|
||||
const proxy_url = https_proxy || http_proxy || all_proxy
|
||||
|
||||
let axins
|
||||
if (proxy_url) {
|
||||
console.log('使用代理:', proxy_url)
|
||||
let ProxyAgent
|
||||
try {
|
||||
ProxyAgent = require('proxy-agent')
|
||||
} catch (e) { // 没执行 npm i proxy-agent
|
||||
ProxyAgent = require('https-proxy-agent')
|
||||
}
|
||||
axins = axios.create({ httpsAgent: new ProxyAgent(proxy_url) })
|
||||
} else {
|
||||
axins = axios.create({})
|
||||
}
|
||||
|
||||
const SA_LOCATION = argv.sa || 'sa'
|
||||
const SA_BATCH_SIZE = 1000
|
||||
const SA_FILES = fs.readdirSync(path.join(__dirname, SA_PATH)).filter(v => v.endsWith('.json'))
|
||||
SA_FILES.flag = 0
|
||||
let SA_TOKENS = get_sa_batch()
|
||||
|
||||
setInterval(() => {
|
||||
SA_FILES.flag = 0
|
||||
SA_TOKENS = get_sa_batch()
|
||||
}, 1000 * 3600 * 12)
|
||||
if (is_pm2()) {
|
||||
setInterval(() => {
|
||||
SA_FILES.flag = 0
|
||||
SA_TOKENS = get_sa_batch()
|
||||
}, 1000 * 3600 * 12)
|
||||
}
|
||||
|
||||
// https://github.com/Leelow/is-pm2/blob/master/index.js
|
||||
function is_pm2 () {
|
||||
return 'PM2_HOME' in process.env || 'PM2_JSON_PROCESSING' in process.env || 'PM2_CLI' in process.env
|
||||
}
|
||||
|
||||
function get_sa_batch () {
|
||||
const new_flag = SA_FILES.flag + SA_BATCH_SIZE
|
||||
@ -33,7 +62,7 @@ function get_sa_batch () {
|
||||
SA_FILES.flag = new_flag
|
||||
return files.map(filename => {
|
||||
const gtoken = new GoogleToken({
|
||||
keyFile: path.join(__dirname, '../sa', filename),
|
||||
keyFile: path.join(__dirname, SA_PATH, filename),
|
||||
scope: ['https://www.googleapis.com/auth/drive']
|
||||
})
|
||||
return { gtoken, expires: 0 }
|
||||
@ -47,6 +76,7 @@ handle_exit(() => {
|
||||
db.prepare('update task set status=? where id=?').run('interrupt', v.id)
|
||||
})
|
||||
records.length && console.log(records.length, 'task interrupted')
|
||||
db.close()
|
||||
})
|
||||
|
||||
async function gen_count_body ({ fid, type, update, service_account }) {
|
||||
@ -71,6 +101,8 @@ async function gen_count_body ({ fid, type, update, service_account }) {
|
||||
return (typeof smy === 'string') ? smy : JSON.stringify(smy)
|
||||
}
|
||||
}
|
||||
const file = await get_info_by_id(fid, service_account)
|
||||
if (file && file.mimeType !== FOLDER_TYPE) return render_smy(summary([file]), type)
|
||||
|
||||
let info, smy
|
||||
const record = db.prepare('SELECT * FROM gd WHERE fid = ?').get(fid)
|
||||
@ -120,7 +152,9 @@ async function count ({ fid, update, sort, type, output, not_teamdrive, service_
|
||||
function get_out_str ({ info, type, sort }) {
|
||||
const smy = summary(info, sort)
|
||||
let out_str
|
||||
if (type === 'html') {
|
||||
if (type === 'tree') {
|
||||
out_str = gen_tree_html(info)
|
||||
} else if (type === 'html') {
|
||||
out_str = make_html(smy)
|
||||
} else if (type === 'json') {
|
||||
out_str = JSON.stringify(smy)
|
||||
@ -164,7 +198,7 @@ function get_all_by_fid (fid) {
|
||||
}
|
||||
|
||||
async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
|
||||
const result = []
|
||||
let result = []
|
||||
const not_finished = []
|
||||
const limit = pLimit(PARALLEL_LIMIT)
|
||||
|
||||
@ -193,7 +227,7 @@ async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
|
||||
should_save && save_files_to_db(parent, files)
|
||||
const folders = files.filter(v => v.mimeType === FOLDER_TYPE)
|
||||
files.forEach(v => v.parent = parent)
|
||||
result.push(...files)
|
||||
result = result.concat(files)
|
||||
return Promise.all(folders.map(v => recur(v.id)))
|
||||
}
|
||||
try {
|
||||
@ -235,15 +269,21 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
|
||||
params.pageSize = Math.min(PAGE_SIZE, 1000)
|
||||
// const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa
|
||||
const use_sa = (fid !== 'root') && service_account
|
||||
const headers = await gen_headers(use_sa)
|
||||
// const headers = await gen_headers(use_sa)
|
||||
// 对于直接子文件数超多的目录(1ctMwpIaBg8S1lrZDxdynLXJpMsm5guAl),可能还没列完,access_token就过期了
|
||||
// 由于需要nextPageToken才能获取下一页的数据,所以无法用并行请求,测试发现每次获取1000个文件的请求大多需要20秒以上才能完成
|
||||
const gtoken = use_sa && (await get_sa_token()).gtoken
|
||||
do {
|
||||
if (pageToken) params.pageToken = pageToken
|
||||
let url = 'https://www.googleapis.com/drive/v3/files'
|
||||
url += '?' + params_to_query(params)
|
||||
const payload = { headers, timeout: TIMEOUT_BASE }
|
||||
let retry = 0
|
||||
let data
|
||||
const payload = { timeout: TIMEOUT_BASE }
|
||||
while (!data && (retry < RETRY_LIMIT)) {
|
||||
const access_token = gtoken ? (await gtoken.getToken()).access_token : (await get_access_token())
|
||||
const headers = { authorization: 'Bearer ' + access_token }
|
||||
payload.headers = headers
|
||||
try {
|
||||
data = (await axins.get(url, payload)).data
|
||||
} catch (err) {
|
||||
@ -258,6 +298,7 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
|
||||
return files
|
||||
}
|
||||
files = files.concat(data.files)
|
||||
argv.sfl && console.log('files.length:', files.length)
|
||||
pageToken = data.nextPageToken
|
||||
} while (pageToken)
|
||||
|
||||
@ -301,7 +342,7 @@ async function get_sa_token () {
|
||||
try {
|
||||
return await real_get_sa_token(tk)
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
console.warn('SA獲取access_token失敗:', e.message)
|
||||
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== tk.gtoken)
|
||||
if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch()
|
||||
}
|
||||
@ -364,9 +405,9 @@ async function create_folder (name, parent, use_sa, limit) {
|
||||
throw new Error(err_message + ' 目錄名:' + name)
|
||||
}
|
||||
|
||||
async function get_name_by_id (fid) {
|
||||
async function get_name_by_id (fid, use_sa) {
|
||||
try {
|
||||
const { name } = await get_info_by_id(fid, true)
|
||||
const { name } = await get_info_by_id(fid, use_sa)
|
||||
return name
|
||||
} catch (e) {
|
||||
return fid
|
||||
@ -379,7 +420,7 @@ async function get_info_by_id (fid, use_sa) {
|
||||
includeItemsFromAllDrives: true,
|
||||
supportsAllDrives: true,
|
||||
corpora: 'allDrives',
|
||||
fields: 'id,name'
|
||||
fields: 'id, name, size, parents, mimeType'
|
||||
}
|
||||
url += '?' + params_to_query(params)
|
||||
const headers = await gen_headers(use_sa)
|
||||
@ -433,7 +474,7 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
|
||||
const record = db.prepare('select * from task where source=? and target=?').get(source, target)
|
||||
if (record) {
|
||||
const copied = db.prepare('select fileid from copied where taskid=?').all(record.id).map(v => v.fileid)
|
||||
const choice = is_server ? 'continue' : await user_choose()
|
||||
const choice = (is_server || argv.yes) ? 'continue' : await user_choose()
|
||||
if (choice === 'exit') {
|
||||
return console.log('退出程序')
|
||||
} else if (choice === 'continue') {
|
||||
@ -474,7 +515,7 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
|
||||
if (min_size) files = files.filter(v => v.size >= min_size)
|
||||
const folders = arr.filter(v => v.mimeType === FOLDER_TYPE)
|
||||
console.log('待複製的目錄數:', folders.length)
|
||||
console.log('待複製的檔案數:', files.length)
|
||||
console.log('待複製的檔案數:', files.length)
|
||||
const mapping = await create_folders({
|
||||
source,
|
||||
folders,
|
||||
@ -513,23 +554,65 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
|
||||
}
|
||||
|
||||
async function copy_files ({ files, mapping, service_account, root, task_id }) {
|
||||
if (!files.length) return
|
||||
console.log('\n開始複製文件,總數:', files.length)
|
||||
const limit = pLimit(PARALLEL_LIMIT)
|
||||
let count = 0
|
||||
|
||||
const loop = setInterval(() => {
|
||||
const now = dayjs().format('HH:mm:ss')
|
||||
const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
|
||||
const message = `${now} | 已複製的檔案數 ${count} | 排隊中檔案數 ${files.length}`
|
||||
print_progress(message)
|
||||
}, 1000)
|
||||
return Promise.all(files.map(async file => {
|
||||
|
||||
let count = 0
|
||||
let concurrency = 0
|
||||
let err
|
||||
do {
|
||||
if (err) {
|
||||
clearInterval(loop)
|
||||
files = null
|
||||
throw err
|
||||
}
|
||||
if (concurrency > PARALLEL_LIMIT) {
|
||||
await sleep(100)
|
||||
continue
|
||||
}
|
||||
const file = files.shift()
|
||||
if (!file) {
|
||||
await sleep(1000)
|
||||
continue
|
||||
}
|
||||
concurrency++
|
||||
const { id, parent } = file
|
||||
const target = mapping[parent] || root
|
||||
const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
|
||||
if (new_file) {
|
||||
count++
|
||||
db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
|
||||
}
|
||||
})).finally(() => clearInterval(loop))
|
||||
copy_file(id, target, service_account, null, task_id).then(new_file => {
|
||||
if (new_file) {
|
||||
count++
|
||||
db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
|
||||
}
|
||||
}).catch(e => {
|
||||
err = e
|
||||
}).finally(() => {
|
||||
concurrency--
|
||||
})
|
||||
} while (concurrency || files.length)
|
||||
return clearInterval(loop)
|
||||
// const limit = pLimit(PARALLEL_LIMIT)
|
||||
// let count = 0
|
||||
// const loop = setInterval(() => {
|
||||
// const now = dayjs().format('HH:mm:ss')
|
||||
// const {activeCount, pendingCount} = limit
|
||||
// const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${activeCount}/排队中${pendingCount}`
|
||||
// print_progress(message)
|
||||
// }, 1000)
|
||||
// return Promise.all(files.map(async file => {
|
||||
// const { id, parent } = file
|
||||
// const target = mapping[parent] || root
|
||||
// const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
|
||||
// if (new_file) {
|
||||
// count++
|
||||
// db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
|
||||
// }
|
||||
// })).finally(() => clearInterval(loop))
|
||||
}
|
||||
|
||||
async function copy_file (id, parent, use_sa, limit, task_id) {
|
||||
@ -558,7 +641,7 @@ async function copy_file (id, parent, use_sa, limit, task_id) {
|
||||
if (message && message.toLowerCase().includes('file limit')) {
|
||||
if (limit) limit.clearQueue()
|
||||
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
|
||||
throw new Error('您的小組雲端硬碟文件數已超限,停止複製')
|
||||
throw new Error(FILE_EXCEED_MSG)
|
||||
}
|
||||
if (use_sa && message && message.toLowerCase().includes('rate limit')) {
|
||||
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken)
|
||||
@ -671,6 +754,22 @@ async function confirm_dedupe ({ file_number, folder_number }) {
|
||||
return answer.value
|
||||
}
|
||||
|
||||
// 需要sa是源文件夹所在盘的manager
|
||||
async function mv_file ({ fid, new_parent, service_account }) {
|
||||
const file = await get_info_by_id(fid, service_account)
|
||||
if (!file) return
|
||||
const removeParents = file.parents[0]
|
||||
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
|
||||
const params = {
|
||||
removeParents,
|
||||
supportsAllDrives: true,
|
||||
addParents: new_parent
|
||||
}
|
||||
url += '?' + params_to_query(params)
|
||||
const headers = await gen_headers(service_account)
|
||||
return axins.patch(url, {}, { headers })
|
||||
}
|
||||
|
||||
// 将文件或文件夹移入回收站,需要 sa 为 content manager 权限及以上
|
||||
async function trash_file ({ fid, service_account }) {
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true`
|
||||
@ -694,7 +793,7 @@ async function rm_file ({ fid, service_account }) {
|
||||
}
|
||||
}
|
||||
|
||||
async function dedupe ({ fid, update, service_account }) {
|
||||
async function dedupe ({ fid, update, service_account, yes }) {
|
||||
let arr
|
||||
if (!update) {
|
||||
const info = get_all_by_fid(fid)
|
||||
@ -707,7 +806,7 @@ async function dedupe ({ fid, update, service_account }) {
|
||||
const dupes = find_dupe(arr)
|
||||
const folder_number = dupes.filter(v => v.mimeType === FOLDER_TYPE).length
|
||||
const file_number = dupes.length - folder_number
|
||||
const choice = await confirm_dedupe({ file_number, folder_number })
|
||||
const choice = yes || await confirm_dedupe({ file_number, folder_number })
|
||||
if (choice === 'no') {
|
||||
return console.log('退出程序')
|
||||
} else if (!choice) {
|
||||
@ -727,7 +826,8 @@ async function dedupe ({ fid, update, service_account }) {
|
||||
file_count++
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('刪除失敗', e.message)
|
||||
console.log('刪除失敗', v)
|
||||
handle_error(e)
|
||||
}
|
||||
}))
|
||||
return { file_count, folder_count }
|
||||
@ -751,4 +851,4 @@ function print_progress (msg) {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id }
|
||||
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id, get_info_by_id, get_access_token, get_sa_token, walk_and_save }
|
||||
|
@ -2,7 +2,7 @@ const Router = require('@koa/router')
|
||||
|
||||
const { db } = require('../db')
|
||||
const { validate_fid, gen_count_body } = require('./gd')
|
||||
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg')
|
||||
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task } = require('./tg')
|
||||
|
||||
const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config')
|
||||
const { tg_whitelist } = AUTH
|
||||
@ -42,16 +42,17 @@ router.get('/api/gdurl/count', async ctx => {
|
||||
router.post('/api/gdurl/tgbot', async ctx => {
|
||||
const { body } = ctx.request
|
||||
console.log('ctx.ip', ctx.ip) // 可以只允许tg服务器的ip
|
||||
console.log('tg message:', body)
|
||||
console.log('tg message:', JSON.stringify(body, null, ' '))
|
||||
if (TG_IPLIST && !TG_IPLIST.includes(ctx.ip)) return ctx.body = 'invalid ip'
|
||||
ctx.body = '' // 早点释放连接
|
||||
const message = body.message || body.edited_message
|
||||
const message_str = JSON.stringify(message)
|
||||
|
||||
const { callback_query } = body
|
||||
if (callback_query) {
|
||||
const { id, data } = callback_query
|
||||
const { id, message, data } = callback_query
|
||||
const chat_id = callback_query.from.id
|
||||
const [action, fid, target] = data.split(' ')
|
||||
const [action, fid, target] = data.split(' ').filter(v => v)
|
||||
if (action === 'count') {
|
||||
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' })
|
||||
counting[fid] = true
|
||||
@ -67,33 +68,45 @@ router.post('/api/gdurl/tgbot', async ctx => {
|
||||
tg_copy({ fid, target: get_target_by_alias(target), chat_id }).then(task_id => {
|
||||
task_id && sm({ chat_id, text: `開始複製,任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
|
||||
}).finally(() => COPYING_FIDS[fid] = false)
|
||||
} else if (action === 'update') {
|
||||
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍等片刻' })
|
||||
counting[fid] = true
|
||||
send_count({ fid, chat_id, update: true }).finally(() => {
|
||||
delete counting[fid]
|
||||
})
|
||||
} else if (action === 'clear_button') {
|
||||
const { message_id, text } = message || {}
|
||||
if (message_id) sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
|
||||
}
|
||||
return reply_cb_query({ id, data }).catch(console.error)
|
||||
}
|
||||
|
||||
const chat_id = message && message.chat && message.chat.id
|
||||
const text = message && message.text && message.text.trim()
|
||||
const text = (message && message.text && message.text.trim()) || ''
|
||||
let username = message && message.from && message.from.username
|
||||
username = username && String(username).toLowerCase()
|
||||
let user_id = message && message.from && message.from.id
|
||||
user_id = user_id && String(user_id).toLowerCase()
|
||||
if (!chat_id || !text || !tg_whitelist.some(v => {
|
||||
if (!chat_id || !tg_whitelist.some(v => {
|
||||
v = String(v).toLowerCase()
|
||||
return v === username || v === user_id
|
||||
})) return console.warn('異常請求')
|
||||
})) {
|
||||
chat_id && sm({ chat_id, text: '您的使用者名稱或ID不在機器人的白名單中,如果是您配置的機器人,請先到config.js中配置自己的username' })
|
||||
return console.warn('收到非白名單用戶的請求')
|
||||
}
|
||||
|
||||
const fid = extract_fid(text) || extract_from_text(text)
|
||||
const fid = extract_fid(text) || extract_from_text(text) || extract_from_text(message_str)
|
||||
const no_fid_commands = ['/task', '/help', '/bm']
|
||||
if (!no_fid_commands.some(cmd => text.startsWith(cmd)) && !validate_fid(fid)) {
|
||||
return sm({ chat_id, text: '未辨識到分享ID' })
|
||||
}
|
||||
if (text.startsWith('/help')) return send_help(chat_id)
|
||||
if (text.startsWith('/bm')) {
|
||||
const [cmd, action, alias, target] = text.split(' ').map(v => v.trim())
|
||||
const [cmd, action, alias, target] = text.split(' ').map(v => v.trim()).filter(v => v)
|
||||
if (!action) return send_all_bookmarks(chat_id)
|
||||
if (action === 'set') {
|
||||
if (!alias || !target) return sm({ chat_id, text: '標籤名和dstID不能為空' })
|
||||
if (alias.length > 24) return sm({ chat_id, text: '標籤名請勿超過24个英文字符' })
|
||||
if (alias.length > 24) return sm({ chat_id, text: '標籤名不要超過24個英文字符長度' })
|
||||
if (!validate_fid(target)) return sm({ chat_id, text: 'dstID格式錯誤' })
|
||||
set_bookmark({ chat_id, alias, target })
|
||||
} else if (action === 'unset') {
|
||||
@ -115,7 +128,7 @@ router.post('/api/gdurl/tgbot', async ctx => {
|
||||
delete counting[fid]
|
||||
}
|
||||
} else if (text.startsWith('/copy')) {
|
||||
let target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim())[1]
|
||||
let target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim()).filter(v => v)[1]
|
||||
target = get_target_by_alias(target) || target
|
||||
if (target && !validate_fid(target)) return sm({ chat_id, text: `目標ID ${target} 格式不正確` })
|
||||
const update = text.endsWith(' -u')
|
||||
@ -126,6 +139,15 @@ router.post('/api/gdurl/tgbot', async ctx => {
|
||||
let task_id = text.replace('/task', '').trim()
|
||||
if (task_id === 'all') {
|
||||
return send_all_tasks(chat_id)
|
||||
} else if (task_id === 'clear') {
|
||||
return clear_tasks(chat_id)
|
||||
} else if (task_id === '-h') {
|
||||
return send_task_help(chat_id)
|
||||
} else if (task_id.startsWith('rm')) {
|
||||
task_id = task_id.replace('rm', '')
|
||||
task_id = parseInt(task_id)
|
||||
if (!task_id) return send_task_help(chat_id)
|
||||
return rm_task({ task_id, chat_id })
|
||||
}
|
||||
task_id = parseInt(task_id)
|
||||
if (!task_id) {
|
||||
@ -134,10 +156,10 @@ router.post('/api/gdurl/tgbot', async ctx => {
|
||||
return running_tasks.forEach(v => send_task_info({ chat_id, task_id: v.id }).catch(console.error))
|
||||
}
|
||||
send_task_info({ task_id, chat_id }).catch(console.error)
|
||||
} else if (text.includes('drive.google.com/') || validate_fid(text)) {
|
||||
return send_choice({ fid: fid || text, chat_id }).catch(console.error)
|
||||
} else if (message_str.includes('drive.google.com/') || validate_fid(text)) {
|
||||
return send_choice({ fid: fid || text, chat_id })
|
||||
} else {
|
||||
sm({ chat_id, text: '暫不支持此命令' })
|
||||
sm({ chat_id, text: '暫不支援此命令' })
|
||||
}
|
||||
})
|
||||
|
||||
|
@ -123,4 +123,4 @@ function format_size (n) {
|
||||
flag++
|
||||
}
|
||||
return n.toFixed(2) + ' ' + units[flag]
|
||||
}
|
||||
}
|
138
src/tg.js
138
src/tg.js
@ -4,12 +4,13 @@ const axios = require('@viegg/axios')
|
||||
const HttpsProxyAgent = require('https-proxy-agent')
|
||||
|
||||
const { db } = require('../db')
|
||||
const { gen_count_body, validate_fid, real_copy, get_name_by_id } = require('./gd')
|
||||
const { gen_count_body, validate_fid, real_copy, get_name_by_id, get_info_by_id, copy_file } = require('./gd')
|
||||
const { AUTH, DEFAULT_TARGET, USE_PERSONAL_AUTH } = require('../config')
|
||||
const { BUTTON_LEVEL } = require('../config_mod')
|
||||
const { tg_token } = AUTH
|
||||
const gen_link = (fid, text) => `<a href="https://drive.google.com/drive/folders/${fid}">${text || fid}</a>`
|
||||
|
||||
if (!tg_token) throw new Error('請先在config.js中設定tg_token')
|
||||
if (!tg_token) throw new Error('請先在config.js裡設定tg_token')
|
||||
const { https_proxy } = process.env
|
||||
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
|
||||
|
||||
@ -18,13 +19,12 @@ const FID_TO_NAME = {}
|
||||
async function get_folder_name (fid) {
|
||||
let name = FID_TO_NAME[fid]
|
||||
if (name) return name
|
||||
name = await get_name_by_id(fid)
|
||||
name = await get_name_by_id(fid, !USE_PERSONAL_AUTH)
|
||||
return FID_TO_NAME[fid] = name
|
||||
}
|
||||
|
||||
function send_help (chat_id) {
|
||||
const text = `<pre>[使用說明]
|
||||
***不支持單檔分享***
|
||||
命令 | 說明
|
||||
=====================
|
||||
/help | 返回本使用說明
|
||||
@ -39,6 +39,11 @@ sourceID可以是共享網址本身,也可以是共享ID。如果命令最后
|
||||
=====================
|
||||
/task taskID(選填) | 返回對應任務的進度信息,若不填taskID則返回所有正在運行的任務進度
|
||||
若填 all 則返回所有任務列表(歷史紀錄)
|
||||
/task | 返回所有正在執行的正在執行的任務詳情
|
||||
/task 7 | 返回ID为 7 的任務詳情
|
||||
/task all | 返回所有任務紀錄列表
|
||||
/task clear | 清除所有狀態為finished的任務紀錄
|
||||
/task rm 7 | 刪除編號為 7 的任務紀錄
|
||||
=====================
|
||||
/bm [action] [alias] [target] | bookmark,添加常用目的資料夾ID
|
||||
會在輸入共享連結後返回的「文件統計」「開始複製」這兩個按鈕的下方出現,方便複製到常用位置。
|
||||
@ -61,9 +66,35 @@ function send_bm_help (chat_id) {
|
||||
return sm({ chat_id, text, parse_mode: 'HTML' })
|
||||
}
|
||||
|
||||
function send_task_help (chat_id) {
|
||||
const text = `<pre>/task [action/id] [id] | 查詢或管理任務進度
|
||||
範例:
|
||||
/task | 返回所有正在執行的正在執行的任務詳情
|
||||
/task 7 | 返回ID为 7 的任務詳情
|
||||
/task all | 返回所有任務紀錄列表
|
||||
/task clear | 清除所有狀態為finished的任務紀錄
|
||||
/task rm 7 | 刪除編號為 7 的任務紀錄
|
||||
</pre>`
|
||||
return sm({ chat_id, text, parse_mode: 'HTML' })
|
||||
}
|
||||
|
||||
function clear_tasks (chat_id) {
|
||||
const finished_tasks = db.prepare('select id from task where status=?').all('finished')
|
||||
finished_tasks.forEach(task => rm_task({ task_id: task.id }))
|
||||
sm({ chat_id, text: '已清除所有狀態為finished的任務紀錄' })
|
||||
}
|
||||
|
||||
function rm_task ({ task_id, chat_id }) {
|
||||
const exist = db.prepare('select id from task where id=?').get(task_id)
|
||||
if (!exist) return sm({ chat_id, text: `不存在任務ID為 ${task_id} 的任務記錄` })
|
||||
db.prepare('delete from task where id=?').run(task_id)
|
||||
db.prepare('delete from copied where taskid=?').run(task_id)
|
||||
if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 記錄` })
|
||||
}
|
||||
|
||||
function send_all_bookmarks (chat_id) {
|
||||
let records = db.prepare('select alias, target from bookmark').all()
|
||||
if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
|
||||
if (!records.length) return sm({ chat_id, text: '數據庫中沒有收藏記錄' })
|
||||
const tb = new Table({ style: { head: [], border: [] } })
|
||||
const headers = ['標籤名', 'dstID']
|
||||
records = records.map(v => [v.alias, v.target])
|
||||
@ -76,7 +107,7 @@ function set_bookmark ({ chat_id, alias, target }) {
|
||||
const record = db.prepare('select alias from bookmark where alias=?').get(alias)
|
||||
if (record) return sm({ chat_id, text: '資料庫中已有同名的收藏' })
|
||||
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
|
||||
return sm({ chat_id, text: `成功設定收藏${alias} | ${target}` })
|
||||
return sm({ chat_id, text: `成功設定收藏:${alias} | ${target}` })
|
||||
}
|
||||
|
||||
function unset_bookmark ({ chat_id, alias }) {
|
||||
@ -91,6 +122,11 @@ function get_target_by_alias (alias) {
|
||||
return record && record.target
|
||||
}
|
||||
|
||||
function get_alias_by_target (target) {
|
||||
const record = db.prepare('select alias from bookmark where target=?').get(target)
|
||||
return record && record.alias
|
||||
}
|
||||
|
||||
function send_choice ({ fid, chat_id }) {
|
||||
if(BUTTON_LEVEL == 1){
|
||||
return sm({
|
||||
@ -104,7 +140,10 @@ function send_choice ({ fid, chat_id }) {
|
||||
[
|
||||
{ text: '開始複製', callback_data: `copy ${fid}` }
|
||||
]
|
||||
].concat(gen_bookmark_choices(fid))
|
||||
].concat(gen_bookmark_choices(fid)).concat([[
|
||||
{ text: '強制更新', callback_data: `update ${fid}` },
|
||||
{ text: '清除按鈕', callback_data: `clear_button` }
|
||||
]])
|
||||
}
|
||||
})
|
||||
}else{
|
||||
@ -117,7 +156,10 @@ function send_choice ({ fid, chat_id }) {
|
||||
{ text: '文件統計', callback_data: `count ${fid}` },
|
||||
{ text: '開始複製', callback_data: `copy ${fid}` }
|
||||
]
|
||||
].concat(gen_bookmark_choices(fid))
|
||||
].concat(gen_bookmark_choices(fid)).concat([[
|
||||
{ text: '強制更新', callback_data: `update ${fid}` },
|
||||
{ text: '清除按鈕', callback_data: `clear_button` }
|
||||
]])
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -131,16 +173,12 @@ function gen_bookmark_choices (fid) {
|
||||
}else{
|
||||
level = BUTTON_LEVEL
|
||||
}
|
||||
const gen_choice = v => ({text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}`})
|
||||
const gen_choice = v => ({ text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}` })
|
||||
const records = db.prepare('select * from bookmark').all()
|
||||
db.close()
|
||||
const result = []
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
for (let i = 0; i < records.length; i += 2) {
|
||||
const line = [gen_choice(records[i])]
|
||||
for(let j = 0; j < level-1; j ++){
|
||||
if (records[i+1]) line.push(gen_choice(records[i+1]))
|
||||
i++
|
||||
}
|
||||
if (records[i + 1]) line.push(gen_choice(records[i + 1]))
|
||||
result.push(line)
|
||||
}
|
||||
return result
|
||||
@ -166,8 +204,8 @@ async function send_all_tasks (chat_id) {
|
||||
// const description = err.response && err.response.data && err.response.data.description
|
||||
// if (description && description.includes('message is too long')) {
|
||||
if (true) {
|
||||
const text = [headers].concat(records).map(v => v.join('\t')).join('\n')
|
||||
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務:\n<pre>${text}</pre>` })
|
||||
const text = [headers].concat(records.slice(-100)).map(v => v.join('\t')).join('\n')
|
||||
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務(僅顯示最近100項):\n<pre>${text}</pre>` })
|
||||
}
|
||||
console.error(err)
|
||||
})
|
||||
@ -176,30 +214,26 @@ async function send_all_tasks (chat_id) {
|
||||
async function get_task_info (task_id) {
|
||||
const record = db.prepare('select * from task where id=?').get(task_id)
|
||||
if (!record) return {}
|
||||
const { source, target, status, copied, mapping, ctime, ftime } = record
|
||||
const { source, target, status, mapping, ctime, ftime } = record
|
||||
const { copied_files } = db.prepare('select count(fileid) as copied_files from copied where taskid=?').get(task_id)
|
||||
const folder_mapping = mapping && mapping.trim().split('\n')
|
||||
const new_folder = folder_mapping && folder_mapping[0].split(' ')[1]
|
||||
const { summary } = db.prepare('select summary from gd where fid=?').get(source) || {}
|
||||
const { file_count, folder_count, total_size } = summary ? JSON.parse(summary) : {}
|
||||
const copied_files = copied ? copied.trim().split('\n').length : 0
|
||||
const total_count = (file_count || 0) + (folder_count || 0)
|
||||
const copied_folders = folder_mapping ? (folder_mapping.length - 1) : 0
|
||||
let text = '任務ID:' + task_id + '\n'
|
||||
let text = '任務ID:' + task_id + ' [' + status + ']\n'
|
||||
const folder_name = await get_folder_name(source)
|
||||
text += '源資料夾:' + gen_link(source, folder_name) + '\n'
|
||||
text += '目的位置:' + gen_link(target) + '\n'
|
||||
text += '目的位置:' + gen_link(target, get_alias_by_target(target)) + '\n'
|
||||
text += '新資料夾:' + (new_folder ? gen_link(new_folder) : '尚未創建') + '\n'
|
||||
text += '任務狀態:' + status + '\n'
|
||||
text += '創建時間:' + dayjs(ctime).format('YYYY-MM-DD HH:mm:ss') + '\n'
|
||||
text += '完成時間:' + (ftime ? dayjs(ftime).format('YYYY-MM-DD HH:mm:ss') : '未完成') + '\n'
|
||||
var pct = copied_folders/(folder_count === undefined ? '未知數量' : folder_count)*100
|
||||
pct = pct.toFixed(2);
|
||||
text += '目錄進度:' + copied_folders + '/' + (folder_count === undefined ? '未知數量' : folder_count) + ' - ' + pct + '%\n'
|
||||
pct = copied_files/(file_count === undefined ? '未知數量' : file_count)*100
|
||||
pct = pct.toFixed(2);
|
||||
text += '文件進度:' + copied_files + '/' + (file_count === undefined ? '未知數量' : file_count) + ' - ' + pct + '%\n'
|
||||
text += '目錄進度:' + copied_folders + '/' + (folder_count === undefined ? '未知數量' : folder_count) + ' - ' + (copied_folders/folder_count*100).toFixed(3) + '%\n'
|
||||
text += '文件進度:' + copied_files + '/' + (file_count === undefined ? '未知數量' : file_count) + ' - ' + (copied_files/file_count*100).toFixed(3) + '%\n'
|
||||
text += '合計大小:' + (total_size || '未知大小')
|
||||
const total_count = (folder_count || 0) + (file_count || 0)
|
||||
return { text, status, total_count }
|
||||
return { text, status, folder_count }
|
||||
}
|
||||
|
||||
async function send_task_info ({ task_id, chat_id }) {
|
||||
@ -213,13 +247,12 @@ async function send_task_info ({ task_id, chat_id }) {
|
||||
} catch (e) {
|
||||
console.log('fail to send message to tg', e.message)
|
||||
}
|
||||
// get_task_info 在task目录数超大时比较吃cpu,如果超1万就不每10秒更新了,以后如果把mapping 也另存一张表可以取消此限制
|
||||
if (!message_id || status !== 'copying' || folder_count > 10000) return
|
||||
// get_task_info 在task目录数超大时比较吃cpu,以后如果最好把mapping也另存一张表
|
||||
if (!message_id || status !== 'copying') return
|
||||
const loop = setInterval(async () => {
|
||||
const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
|
||||
const { text, status } = await get_task_info(task_id)
|
||||
if (status !== 'copying') clearInterval(loop)
|
||||
axins.post(url, { chat_id, message_id, text, parse_mode: 'HTML' }).catch(e => console.error(e.message))
|
||||
sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
|
||||
}, 10 * 1000)
|
||||
}
|
||||
|
||||
@ -229,6 +262,14 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
|
||||
sm({ chat_id, text: '請輸入目的地ID或先在config.js中設定預設複製的目的地ID(DEFAULT_TARGET)' })
|
||||
return
|
||||
}
|
||||
const file = await get_info_by_id(fid, !USE_PERSONAL_AUTH)
|
||||
if (file && file.mimeType !== 'application/vnd.google-apps.folder') {
|
||||
return copy_file(fid, target, !USE_PERSONAL_AUTH).then(data => {
|
||||
sm({ chat_id, parse_mode: 'HTML', text: `單檔複製成功,位置:${gen_link(target)}` })
|
||||
}).catch(e => {
|
||||
sm({ chat_id, text: `單檔複製成功失敗,失敗訊息:${e.message}` })
|
||||
})
|
||||
}
|
||||
|
||||
let record = db.prepare('select id, status from task where source=? and target=?').get(fid, target)
|
||||
if (record) {
|
||||
@ -242,7 +283,7 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
|
||||
|
||||
real_copy({ source: fid, update, target, service_account: !USE_PERSONAL_AUTH, is_server: true })
|
||||
.then(async info => {
|
||||
if (!record) record = {} // 防止无限循环
|
||||
if (!record) record = {} // 防止無限循環
|
||||
if (!info) return
|
||||
const { task_id } = info
|
||||
const { text } = await get_task_info(task_id)
|
||||
@ -250,14 +291,11 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
|
||||
})
|
||||
.catch(err => {
|
||||
const task_id = record && record.id
|
||||
if (task_id){
|
||||
db.prepare('update task set status=? where id=?').run('error', task_id)
|
||||
db.close()
|
||||
}
|
||||
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
|
||||
if (!record) record = {}
|
||||
console.error('複製失敗', fid, '-->', target)
|
||||
console.error(err)
|
||||
sm({ chat_id, text: '複製失敗,失敗訊息:' + err.message })
|
||||
sm({ chat_id, text: (task_id || '') + '複製失敗,失敗訊息:' + err.message })
|
||||
})
|
||||
|
||||
while (!record) {
|
||||
@ -310,18 +348,21 @@ ${table}</pre>`
|
||||
文件總數:${file_count}
|
||||
目錄總數:${folder_count}
|
||||
合計大小:${total_size}
|
||||
</pre>`
|
||||
</pre>`
|
||||
})
|
||||
}
|
||||
throw err
|
||||
})
|
||||
}
|
||||
|
||||
function sm (data) {
|
||||
const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
|
||||
function sm (data, endpoint) {
|
||||
endpoint = endpoint || 'sendMessage'
|
||||
const url = `https://api.telegram.org/bot${tg_token}/${endpoint}`
|
||||
return axins.post(url, data).catch(err => {
|
||||
// console.error('fail to post', url, data)
|
||||
console.error('fail to send message to tg:', err.message)
|
||||
const err_data = err.response && err.response.data
|
||||
err_data && console.error(err_data)
|
||||
})
|
||||
}
|
||||
|
||||
@ -333,9 +374,11 @@ function extract_fid (text) {
|
||||
if (!text.startsWith('http')) text = 'https://' + text
|
||||
const u = new URL(text)
|
||||
if (u.pathname.includes('/folders/')) {
|
||||
const reg = /[^/?]+$/
|
||||
const match = u.pathname.match(reg)
|
||||
return match && match[0]
|
||||
return u.pathname.split('/').map(v => v.trim()).filter(v => v).pop()
|
||||
} else if (u.pathname.includes('/file/')) {
|
||||
const file_reg = /file\/d\/([a-zA-Z0-9_-]+)/
|
||||
const file_match = u.pathname.match(file_reg)
|
||||
return file_match && file_match[1]
|
||||
}
|
||||
return u.searchParams.get('id')
|
||||
} catch (e) {
|
||||
@ -344,9 +387,10 @@ function extract_fid (text) {
|
||||
}
|
||||
|
||||
function extract_from_text (text) {
|
||||
const reg = /https?:\/\/drive.google.com\/[^\s]+/g
|
||||
// const reg = /https?:\/\/drive.google.com\/[^\s]+/g
|
||||
const reg = /https?:\/\/drive.google.com\/[a-zA-Z0-9_\\/?=&-]+/g
|
||||
const m = text.match(reg)
|
||||
return m && extract_fid(m[0])
|
||||
}
|
||||
|
||||
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark }
|
||||
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task }
|
||||
|
115
src/tree.js
Normal file
115
src/tree.js
Normal file
@ -0,0 +1,115 @@
|
||||
module.exports = { gen_tree_html }
|
||||
|
||||
function gen_tree_html (arr) {
|
||||
const data = gen_tree_data(arr, is_gd_folder)
|
||||
return tree_tpl(JSON.stringify(data))
|
||||
}
|
||||
|
||||
function tree_tpl (str) {
|
||||
return `<!doctype html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no">
|
||||
<meta name="theme-color" content="#000000">
|
||||
<title>Folder Tree</title>
|
||||
<link href="https://cdn.jsdelivr.net/gh/iwestlin/gd-utils/static/tree.min.css" rel="stylesheet">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<!-- source code: https://github.com/iwestlin/foldertree/blob/master/app.jsx -->
|
||||
<div id="root"></div>
|
||||
<script type="text/javascript">
|
||||
var treedata = ${str}
|
||||
</script>
|
||||
<script type="text/javascript" src="https://cdn.jsdelivr.net/gh/iwestlin/gd-utils/static/tree.min.js"></script>
|
||||
</body>
|
||||
|
||||
</html>`
|
||||
}
|
||||
|
||||
function is_gd_folder (data) {
|
||||
return data.mimeType === 'application/vnd.google-apps.folder'
|
||||
}
|
||||
|
||||
function gen_tree_data (data, is_folder) {
|
||||
if (!data || !data.length) return []
|
||||
const folders = data.filter(is_folder)
|
||||
const files = data.filter(v => !is_folder(v))
|
||||
const total_size = sum(files.map(v => v.size))
|
||||
const root = {
|
||||
title: `/根目录 [共${files.length} 个文件(不包括文件夹), ${format_size(total_size)}]`,
|
||||
key: data[0].parent
|
||||
}
|
||||
if (!folders.length) return [root]
|
||||
const sub_folders = folders.filter(v => v.parent === folders[0].parent)
|
||||
sub_folders.forEach(v => {
|
||||
sum_files(v, data, is_folder)
|
||||
count_files(v, data, is_folder)
|
||||
})
|
||||
sort_folders(folders, 'count')
|
||||
sort_folders(sub_folders, 'count')
|
||||
folders.forEach(v => {
|
||||
let {name, size, count, id} = v
|
||||
if (name.length > 50) name = name.slice(0, 48) + '...'
|
||||
v.title = `${name} | [共${count}个文件 ${format_size(size)}]`
|
||||
})
|
||||
root.children = sub_folders.map(v => gen_node(v, folders))
|
||||
return [root]
|
||||
}
|
||||
|
||||
function sort_folders (folders, type) {
|
||||
if (!folders || !folders.length) return
|
||||
if (type === 'size') return folders.sort((a, b) => b.size - a.size)
|
||||
if (type === 'count') return folders.sort((a, b) => b.count - a.count)
|
||||
}
|
||||
|
||||
function gen_node (v, folders) {
|
||||
const {id, title, node} = v
|
||||
if (node) return node
|
||||
return v.node = {
|
||||
title,
|
||||
key: id,
|
||||
children: v.children || folders.filter(vv => vv.parent === id).map(vv => gen_node(vv, folders))
|
||||
}
|
||||
}
|
||||
|
||||
function count_files (folder, arr, is_folder) {
|
||||
if (folder.count) return folder.count
|
||||
const children = arr.filter(v => v.parent === folder.id)
|
||||
return folder.count = sum(children.map(v => {
|
||||
if (is_folder(v)) return count_files(v, arr, is_folder)
|
||||
return 1
|
||||
}))
|
||||
}
|
||||
|
||||
function sum_files (folder, arr, is_folder) {
|
||||
if (folder.size) return folder.size
|
||||
const children = arr.filter(v => v.parent === folder.id)
|
||||
return folder.size = sum(children.map(v => {
|
||||
if (is_folder(v)) return sum_files(v, arr, is_folder)
|
||||
return v.size
|
||||
}))
|
||||
}
|
||||
|
||||
function sum (arr) {
|
||||
let result = 0
|
||||
for (const v of arr) {
|
||||
result += Number(v) || 0
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function format_size (n) {
|
||||
n = Number(n)
|
||||
if (Number.isNaN(n)) return ''
|
||||
if (n < 0) return 'invalid size'
|
||||
const units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
||||
let flag = 0
|
||||
while (n >= 1024) {
|
||||
n = n / 1024
|
||||
flag++
|
||||
}
|
||||
return n.toFixed(2) + ' ' + units[flag]
|
||||
}
|
BIN
static/colab.png
Normal file
BIN
static/colab.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 208 KiB |
1
static/tree.min.css
vendored
Normal file
1
static/tree.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
1
static/tree.min.js
vendored
Normal file
1
static/tree.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
static/tree.png
Normal file
BIN
static/tree.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 176 KiB |
@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const { argv } = require('yargs')
|
||||
.usage('用法: ./$0 folder-id [options]\nfolder-id 是你想检测SA是否对其有阅读权限的目录ID')
|
||||
.usage('用法: ./$0 folder-id\nfolder-id 是你想檢測SA是否對其有閱讀權限的目錄ID')
|
||||
.help('h')
|
||||
.alias('h', 'help')
|
||||
|
||||
@ -28,18 +28,18 @@ main()
|
||||
async function main () {
|
||||
const [fid] = argv._
|
||||
if (validate_fid(fid)) {
|
||||
console.log('开始检测', SA_TOKENS.length, '个SA帐号')
|
||||
console.log('開始檢測', SA_TOKENS.length, '個SA帳號')
|
||||
const invalid_sa = await get_invalid_sa(SA_TOKENS, fid)
|
||||
if (!invalid_sa.length) return console.log('已检测', SA_TOKENS.length, '个SA,未检测到无效帐号')
|
||||
if (!invalid_sa.length) return console.log('已檢測', SA_TOKENS.length, '個SA,未檢測到無效帳號')
|
||||
const choice = await choose(invalid_sa.length)
|
||||
if (choice === 'yes') {
|
||||
mv_sa(invalid_sa)
|
||||
console.log('成功移动')
|
||||
console.log('成功移動')
|
||||
} else {
|
||||
console.log('成功退出,无效的SA记录:', invalid_sa)
|
||||
console.log('成功退出,無效的SA記錄:', invalid_sa)
|
||||
}
|
||||
} else {
|
||||
console.warn('目录ID缺失或格式错误')
|
||||
console.warn('目錄ID缺失或格式錯誤')
|
||||
}
|
||||
}
|
||||
|
||||
@ -55,9 +55,9 @@ async function choose (count) {
|
||||
const answer = await prompts({
|
||||
type: 'select',
|
||||
name: 'value',
|
||||
message: `检测到 ${count} 个无效的SA,是否将它们移动到 sa/invalid 目录下?`,
|
||||
message: `檢測到 ${count} 個無效的SA,是否將它們移動到 sa/invalid 目錄下?`,
|
||||
choices: [
|
||||
{ title: 'Yes', description: '确认移动', value: 'yes' },
|
||||
{ title: 'Yes', description: '確認移動', value: 'yes' },
|
||||
{ title: 'No', description: '不做更改,直接退出', value: 'no' }
|
||||
],
|
||||
initial: 0
|
||||
@ -66,30 +66,40 @@ async function choose (count) {
|
||||
}
|
||||
|
||||
async function get_invalid_sa (arr, fid) {
|
||||
if (!fid) throw new Error('请指定要检测权限的目录ID')
|
||||
if (!fid) throw new Error('請指定要檢測權限的目錄ID')
|
||||
const fails = []
|
||||
let flag = 0
|
||||
let good = 0
|
||||
for (const v of arr) {
|
||||
console.log('检测进度', `${flag++}/${arr.length}`)
|
||||
console.log('正常/异常', `${good}/${fails.length}`)
|
||||
console.log('檢測進度', `${flag++}/${arr.length}`)
|
||||
console.log('正常/異常', `${good}/${fails.length}`)
|
||||
const {gtoken, filename} = v
|
||||
try {
|
||||
const access_token = await get_sa_token(gtoken)
|
||||
await get_info(fid, access_token)
|
||||
good++
|
||||
} catch (e) {
|
||||
handle_error(e)
|
||||
const status = e && e.response && e.response.status
|
||||
if (Number(status) === 400) fails.push(filename) // access_token 获取失败
|
||||
|
||||
const data = e && e.response && e.response.data
|
||||
const code = data && data.error && data.error.code
|
||||
if (Number(code) === 404) fails.push(filename) // 读取文件夹信息失败
|
||||
if ([404, 403].includes(Number(code))) fails.push(filename) // 读取文件夹信息失败
|
||||
}
|
||||
}
|
||||
return fails
|
||||
}
|
||||
|
||||
function handle_error (err) {
|
||||
const data = err && err.response && err.response.data
|
||||
if (data) {
|
||||
console.error(JSON.stringify(data))
|
||||
} else {
|
||||
console.error(err.message)
|
||||
}
|
||||
}
|
||||
|
||||
async function get_info (fid, access_token) {
|
||||
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
|
||||
let params = {
|
||||
|
Loading…
Reference in New Issue
Block a user