This commit is contained in:
DennesUang 2021-06-10 16:15:16 +08:00 committed by GitHub
commit eb09795713
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 775 additions and 157 deletions

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
node_modules/
gdurl.sqlite
gdurl.sqlite*
config.js
sa/*.json
backup/*.sqlite

14
backup-db.js Normal file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env node
const path = require('path')
const {db} = require('./db')
const filepath = path.join(__dirname, 'backup', `${Date.now()}.sqlite`)
db.backup(filepath)
.then(() => {
console.log(filepath)
})
.catch((err) => {
console.log('backup failed:', err)
})

0
backup/.keep Normal file
View File

28
bookmark.js Normal file
View File

@ -0,0 +1,28 @@
const fs = require('fs')
const {db} = require('./db')
const action = process.argv[2] || 'export'
const filepath = process.argv[3] || 'bookmarks.json'
if (action === 'export') {
const bookmarks = db.prepare('select * from bookmark').all()
fs.writeFileSync(filepath, JSON.stringify(bookmarks))
console.log('bookmarks exported', filepath)
} else if (action === 'import') {
let bookmarks = fs.readFileSync(filepath, 'utf8')
bookmarks = JSON.parse(bookmarks)
bookmarks.forEach(v => {
const {alias, target} = v
const exist = db.prepare('select alias from bookmark where alias=?').get(alias)
if (exist) {
db.prepare('update bookmark set target=? where alias=?').run(target, alias)
} else {
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
}
})
console.log('bookmarks imported', bookmarks)
} else {
console.log('[help info]')
console.log('export: node bookmark.js export bm.json')
console.log('import: node bookmark.js import bm.json')
}

8
clear-db.js Normal file
View File

@ -0,0 +1,8 @@
const { db } = require('./db')
const record = db.prepare('select count(*) as c from gd').get()
db.prepare('delete from gd').run()
console.log('已刪除', record.c, '項資料')
db.exec('vacuum')
db.close()

View File

@ -1,27 +1,24 @@
// 单次请求多少毫秒未响应以后超时(基准值,若连续超时则下次调整为上次的2倍
// 單次請求多少毫秒無回應以後超時(基準值,若連續超時則下次調整為上次的2倍
const TIMEOUT_BASE = 7000
// 最大超时设置比如某次请求第一次7s超时第二次14s第三次28s第四次56s第五次不是112s而是60s后续同理
// 最大超時設置比如某次請求第一次7s超時第二次14s第三次28s第四次56s第五次不是112s而是60s後續同理
const TIMEOUT_MAX = 60000
const LOG_DELAY = 5000 // 日志输出时间间隔,单位毫秒
const PAGE_SIZE = 1000 // 每次网络请求读取目录下的文件数,数值越大,越有可能超时,不得超过1000
const LOG_DELAY = 5000 // 日志輸出時間間隔,單位毫秒
const PAGE_SIZE = 1000 // 每次網絡請求讀取目錄下的文件數,數值越大,越有可能超時,不得超過1000
const RETRY_LIMIT = 7 // 如果某次请求失败,允许其重试的最大次数
const PARALLEL_LIMIT = 20 // 网络请求的并行数量,可根据网络环境调整
const RETRY_LIMIT = 7 // 如果某次請求失敗,允許其重試的最大次數
const PARALLEL_LIMIT = 20 // 網絡請求的並行數量,可根據網絡環境調整(即多線程之線程數量)
const DEFAULT_TARGET = '' // 必填,拷贝默认目的地ID如果不指定target则会复制到此处建议填写团队盘ID
const DEFAULT_TARGET = '' // 必填,copy時預設的dstID建議填寫小組雲端硬碟ID
const AUTH = { // 如果您拥有service account的json授权文件可将其拷贝至 sa 目录中以代替 client_id/secret/refrest_token
const AUTH = { // 如果您擁有SA的json授權文件可將其拷貝至 sa 目錄中以代替 client_id/secret/refrest_token 這裡建議使用自己的client_id, 具體參考說明文件#個人帳號配置
client_id: 'your_client_id',
client_secret: 'your_client_secret',
refresh_token: 'your_refrest_token',
expires: 0, // 可以留空
access_token: '', // 可以留空
tg_token: 'bot_token', // 你的 telegram robot 的 token获取方法参见 https://core.telegram.org/bots#6-botfather
tg_whitelist: ['your_tg_username'] // 你的tg username(t.me/username)bot只会执行这个列表里的用户所发送的指令
tg_token: 'bot_token', // 你的 telegram robot 的 token獲取方法參見 https://core.telegram.org/bots#6-botfather
tg_whitelist: ['your_tg_username'] // 你的tg username(t.me/username)bot只會執行這個列表中的用戶所發送的指令
}
//-------------------MOD-------------------
const SA_PATH = '../sa' //sa路徑配置, 給定絕對路徑或是以src為當前路徑給定相對路徑, 預設為'../sa'
const BUTTON_LEVEL = 1 //預設為1, 填入大於2皆視為2
module.exports = { AUTH, PARALLEL_LIMIT, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET, SA_PATH, BUTTON_LEVEL }
module.exports = { AUTH, PARALLEL_LIMIT, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET }

5
config_mod.js Normal file
View File

@ -0,0 +1,5 @@
//-------------------MOD-------------------
const SA_PATH = '../sa' //sa路徑配置, 給定絕對路徑或是以src為當前路徑給定相對路徑, 預設為'../sa'
const BUTTON_LEVEL = 1 //預設為1, 填入大於2皆視為2
module.exports = { SA_PATH, BUTTON_LEVEL }

24
copy
View File

@ -5,19 +5,21 @@ const bytes = require('bytes')
const { argv } = require('yargs')
.usage('用法: ./$0 <source id> <target id> [options]\ntarget id可选不填则使用config.js里的DEFAULT_TARGET')
.alias('u', 'update')
.describe('u', '不使用本地缓存,强制从线上获取源文件夹信息')
.describe('u', '不使用本地快取,則無視快取記錄強制從線上獲取源資料夾資訊')
.alias('y', 'yes')
.describe('yes', '如果發現拷貝紀錄,直接繼續上次的進度')
.alias('f', 'file')
.describe('f', '复制单个文件')
.describe('f', '複製單一文件')
.alias('n', 'name')
.describe('n', '给目标文件夹重命名,不填则保留原始目录名')
.describe('n', '給目標資料夾重新命名,不填則保留原始目錄名')
.alias('N', 'not_teamdrive')
.describe('N', '如果不是团队盘链接,可以加上此参数以提高接口查询效率,降低延迟')
.describe('N', '如果不是小組雲端硬碟連結,可以加上此参數以提高接口查詢效率,降低延遲')
.alias('s', 'size')
.describe('s', '不填默认拷贝全部文件如果设置了这个值则过滤掉小于这个size的文件必须以b结比如10mb')
.describe('s', '不填則預設拷貝全部文件如果設置了这個值則過濾掉小於这個大小的檔案必須以b為結比如10mb')
.alias('S', 'service_account')
.describe('S', '指定使用service account进行操作,前提是必须在 ./sa 目录下放置json授权文件请确保sa帐号拥有操作权限。')
.describe('S', '指定使用service account進行操作,前提是必須在 ./sa 目錄下放置json授權文件請確認sa帳號擁有相關操作權限。')
.alias('D', 'dncnr')
.describe('D', 'do not create new root, 不在目的地创建同名文件夹,直接将源文件夹中的文件原样复制到目的文件夹中')
.describe('D', 'do not create new root, 不在目的地創建同名資料夾,直接將來源資料夾中的文件原樣複製到目的資料夾中')
.help('h')
.alias('h', 'help')
@ -33,19 +35,19 @@ if (validate_fid(source)) {
if (!validate_fid(target)) throw new Error('target id 格式不正确')
return copy_file(source, target, service_account).then(r => {
const link = 'https://drive.google.com/drive/folders/' + target
console.log('任完成,文件所在位置:\n', link)
console.log('任完成,文件所在位置:\n', link)
}).catch(console.error)
}
let min_size
if (size) {
console.log(`不复制大小低于 ${size} 的文件`)
console.log(`不複製大小低於 ${size} 的文件`)
min_size = bytes.parse(size)
}
copy({ source, target, name, min_size, update, not_teamdrive, service_account, dncnr }).then(folder => {
if (!folder) return
const link = 'https://drive.google.com/drive/folders/' + folder.id
console.log('\n任务完成,新文件夹链接\n', link)
console.log('\n任務完成,新資料夾連結\n', link)
})
} else {
console.warn('目录ID缺失或格式错误')
console.warn('無目錄ID或格式錯誤')
}

24
count
View File

@ -1,23 +1,23 @@
#!/usr/bin/env node
const { argv } = require('yargs')
.usage('用法: ./$0 <目ID> [options]')
.example('./$0 1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75', '获取 https://drive.google.com/drive/folders/1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75 内包含的的所有文件的统计信息')
.example('./$0 root -s size -t html -o out.html', '获取个人盘根目录统计信息结果以HTML表格输出根据总大小逆序排列保存到本目录下的out.html文件中不存在则新建存在则覆盖')
.example('./$0 root -s name -t json -o out.json', '获取个人盘根目录统计信息结果以JSON格式输出根据文件扩展名排序保存到本目录下的out.json文件中')
.example('./$0 root -t all -o all.json', '获取个人盘根目录统计信息将所有文件信息包括文件夹以JSON格式输出保存到本目录下的all.json文件中')
.usage('用法: ./$0 <目ID> [options]')
.example('./$0 1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75', '獲取 https://drive.google.com/drive/folders/1ULY8ISgWSOVc0UrzejykVgXfVL_I4r75 内包含的所有文件的統計資訊')
.example('./$0 root -s size -t html -o out.html', '獲取個人空間根目錄統計資訊結果以HTML表格輸出根據總大小降冪排列保存到本目錄下的out.html文件中不存在則新建存在的覆蓋')
.example('./$0 root -s name -t json -o out.json', '獲取個人空間根目錄統計資訊結果以JSON格式輸出根據文件扩展名排序保存到本目录下的out.json文件中')
.example('./$0 root -t all -o all.json', '獲取個人空間根目錄統計資訊將所有文件資訊包括資料夾以JSON格式輸出保存到本目錄下的all.json文件中')
.alias('u', 'update')
.describe('u', '强制从线上获取信息(无视是否存在本地缓存)')
.describe('u', '強制從線上獲取資訊(不論是否存在本地快取)')
.alias('N', 'not_teamdrive')
.describe('N', '如果不是团队盘链接,可以加上此参数以提高接口查询效率,降低延迟。如果要统计的是个人盘且./sa目录下的service account没有相关权限请确保加上此参数以使用个人的auth信息进行查询')
.describe('N', '如果不是小組雲端硬碟連結,可以加上此参數以提高接口查詢效率,降低延遲。如果要統計的是帳號下之個人空間,且./sa目錄下的service account沒有相關權限請加上此參數以使用個人的auth身份進行查詢')
.alias('S', 'service_account')
.describe('S', '指定使用service account进行统计前提是必须在sa目录下放置SA json文件')
.describe('S', '指定使用service account進行統計前提是必須在sa路徑下放置SA json文件')
.alias('s', 'sort')
.describe('s', '统计结果排序方法,可选值 name 或 size不填则默认根据文件数量逆序排列')
.describe('s', '統計結果排序方法,可選值如下: name 或 size不填則預設根據文件數量降冪排列')
.alias('t', 'type')
.describe('t', '统计结果输出类型,可选值 html/json/allall表示输出所有文件json数据最好搭配 -o 使用。不填则默认输出命令行表格')
.describe('t', '統計結果輸出類型,可選值如下: html/tree/json/allall表示輸出所有文件json數據最好與 -o 一起使用。不填則預設輸出命令列表格')
.alias('o', 'output')
.describe('o', '统计结果输出文件,适合搭配 -t 使用')
.describe('o', '統計結果輸出文件,適合與 -t 一起使用')
.help('h')
.alias('h', 'help')
@ -27,5 +27,5 @@ if (validate_fid(fid)) {
const { update, sort, type, output, not_teamdrive, service_account } = argv
count({ fid, update, sort, type, output, not_teamdrive, service_account }).catch(console.error)
} else {
console.warn('目录ID缺失或格式错误')
console.warn('無目錄ID或格式錯誤')
}

12
dedupe
View File

@ -1,11 +1,13 @@
#!/usr/bin/env node
const { argv } = require('yargs')
.usage('用法: ./$0 <source id> <target id> [options]')
.usage('用法: ./$0 <folder-id> [options]')
.alias('y', 'yes')
.describe('yes', '如果發現重複項目,直接刪除')
.alias('u', 'update')
.describe('u', '不使用本地缓存,强制从线上获取源文件夹信息')
.describe('u', '不使用本地快取,則無視快取記錄強制從線上獲取源資料夾資訊')
.alias('S', 'service_account')
.describe('S', '使用service account进行操作,前提是必须在 ./sa 目录下放置sa授权json文件')
.describe('S', '使用service account進行操作,前提是必須在 ./sa 目錄下存放sa授權json文件')
.help('h')
.alias('h', 'help')
@ -17,8 +19,8 @@ if (validate_fid(fid)) {
dedupe({ fid, update, service_account }).then(info => {
if (!info) return
const { file_count, folder_count } = info
console.log('任务完成,共删除文件数:', file_count, '目录数', folder_count)
console.log('任務完成,共刪除檔案數:', file_count, '目錄數', folder_count)
})
} else {
console.warn('目录ID缺失或格式错误')
console.warn('無目錄ID或格式錯誤')
}

245
gdutilsinstall_cf.sh Normal file
View File

@ -0,0 +1,245 @@
#!/bin/bash
echo
echo -e "\033[1;32m===== <<gdutils项目一件部署脚本要求及说明>> =====\033[0m"
echo -e "\033[1;32m---------------[ v2.1 by oneking ]---------------\033[0m"
echo -e "\033[32m 01.\033[0m 本脚本是针对TG大神@viegg的gdutils项目一键部署脚本;"
echo -e "\033[32m 02.\033[0m 脚本包括“TD盘VPS上查询转存部署”和“Telegram机器人部署”两部分"
echo -e "\033[32m 03.\033[0m 本脚本适应CentOS/Debian/Ubuntu三种操作系统自动识别、自动选择对应分支一键安装部署"
echo -e "\033[32m 04.\033[0m 三步即可完成部署上传脚本到VPS → 设置脚本执行权限 → 运行"
echo -e "\033[32m 05.\033[0m 准备工作一在TG上注册好机器人取得并记录下该机器人TOKEN"
echo -e "\033[32m 06.\033[0m 准备工作二拥有一个域名绑定到cloudflare解析到该机器人所在服务器IP"
echo -e "\033[32m 07.\033[0m 准备工作三:向机器人@userinfobot获取个人TG账号ID并记录"
echo -e "\033[32m 08.\033[0m 准备工作四注册好一个Google team drive加入sa并记录下该盘ID"
echo -e "\033[32m 09.\033[0m 经测试可用完美安装系统Centos 7/8 debian 9/10 ubuntu 16.04/18.04/19.10/20.04"
echo -e "\033[32m 10.\033[0m 部署过程中有任何问题请把“错误截图”“部署VPS系统名称版本”信息发给TGonekings 或 vitaminor@gmail.com"
echo -e "\033[1;32m------------------------------------------------\033[0m"
read -s -n1 -p "★★★ 如已做好以上[5/6/7/8]准备或不需要安装Telegram机器人请按任意键开始部署如未做好准备请按“Ctrl+c”终止脚本 ★★★"
echo
echo -e "\033[1;32m------------------------------------------------\033[0m"
# 识别操作系统
aNAME="$(uname -a)"
bNAME="$(cat /proc/version)"
cNAME="$(lsb_release -a)"
if [ -f "/etc/redhat-release" ]; then
if [[ $(cat /etc/redhat-release) =~ "CentOS" ]]; then
os="CentOS"
fi
elif [ "$aNAME"=~"Debian" -o "$bNAME"=~"Debian" -o "$cNAME"=~"Debian" ]; then
os="Debian"
elif [ "$aNAME"=~"Ubuntu" -o "$bNAME"=~"Ubuntu" -o "$cNAME"=~"Ubuntu" ]; then
os="Debian"
elif [ "$aNAME"=~"CentOS" -o "$bNAME"=~"CentOS" -o "$cNAME"=~"CentOS" ]; then
os="CentOS"
elif [ "$aNAME"=~"Darwin" -o "$bNAME"=~"Darwin" -o "$cNAME"=~"Darwin" ]; then
os="mac"
else
os="$bNAME"
fi
# 需要安装的软件工具及依赖
insofts=(epel-release update upgrade wget curl git unzip zip python3-distutils python3 python3-pip)
#根据操作系统设置变量
if [[ "$os" = "Debian" ]]; then
cmd_install="apt-get" #安装命令
cmd_install_rely="build-essential" #c++编译环境
nodejs_curl="https://deb.nodesource.com/setup_10.x" #nodejs下载链接
cmd_install_rpm_build="" #安装rpm-build
nginx_conf="/etc/nginx/sites-enabled/" #nginx配置文件存放路径
rm_nginx_default="rm -f /etc/nginx/sites-enabled/default" #删除default
echo
echo -e "\033[1;32m★★★★★ 您的操作系统为Debian即将为你开始部署gdutils项目 ★★★★★\033[0m"
elif [[ "$os" = "Ubuntu" ]]; then
cmd_install="sudo apt-get"
cmd_install_rely="build-essential"
nodejs_curl="https://deb.nodesource.com/setup_10.x"
cmd_install_rpm_build=""
nginx_conf="/etc/nginx/sites-enabled/"
rm_nginx_default="rm -f /etc/nginx/sites-enabled/default"
echo
echo -e "\033[1;32m★★★★★ 您的操作系统为Ubuntu即将为你开始部署gdutils项目 ★★★★★\033[0m"
elif [[ "$os" = "CentOS" ]]; then
cmd_install="yum"
cmd_install_rely="gcc-c++ make"
nodejs_curl="https://rpm.nodesource.com/setup_10.x"
cmd_install_rpm_build="yum install rpm-build -y"
nginx_conf="/etc/nginx/conf.d/"
rm_nginx_default=""
echo
echo -e "\033[1;32m★★★★★ 您的操作系统为Centos即将为你开始部署gdutils项目 ★★★★★\033[0m"
elif [[ "$os" = "mac" ]]; then
echo
echo -e "\033[1;32m★★★★★ 您的操作系统为MacOS请在图形界面手动安装 ★★★★★\033[0m"
exit
echo
echo
else
echo
echo -e "\033[1;32m unknow os $OS, exit! \033[0m"
exit
echo
echo
fi
echo
echo -e "\033[1;32m===== <<升级系统/更新软件/安装工具/安装依赖>> =====\033[0m"
echo
#安装which和sudo
if [[ "$(which which)" == "" ]]; then
echo -e "\033[1;32m“which”开始安装......\033[0m"
$cmd_install install which -y
echo -e "\033[1;32m------------------------------------------------\033[0m"
elif [[ "$(which sudo)" == "" ]]; then
echo -e "\033[1;32m“sudo”开始安装......\033[0m"
$cmd_install install sudo -y
echo -e "\033[1;32m------------------------------------------------\033[0m"
fi
#安装工具和依赖
for ((aloop = 0; aloop < ${#insofts[@]}; aloop++)); do
if [ ${insofts[$aloop]} = "update" -o ${insofts[$aloop]} = "upgrade" ]; then
echo -e "\033[1;32m“${insofts[$aloop]}”开始安装......\033[0m"
$cmd_install ${insofts[$aloop]} -y
echo -e "\033[1;32m------------------------------------------------\033[0m"
else
echo -e "\033[1;32m“${insofts[$aloop]}”开始安装......\033[0m"
$cmd_install install ${insofts[$aloop]} -y
echo -e "\033[1;32m------------------------------------------------\033[0m"
fi
done
echo
echo -e "\033[1;32m===== <<安装gdutils依赖-nodejs和npm/安装配置gdutils>> =====\033[0m"
echo
$cmd_install install $cmd_install_rely -y
curl -sL $nodejs_curl | bash -
$cmd_install install nodejs -y
$cmd_install_rpm_build
git clone https://github.com/liaojack8/gd-utils-cht && cd gd-utils-cht
npm config set unsafe-perm=true
npm i
echo
echo -e "\033[1;32m★★★ 恭喜您!gdutils统计转存系统已经正确安装完成请上传sa到“./gd-utils-cht/sa/”目录下完成最后的配置 ★★★\033[0m"
echo
#################################################################################################
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
read -s -n1 -p "★★★ 下面将部署Telegram机器人请确保准备所需条件已准备好按任意键开始部署机器人如未做好准备请按“Ctrl+c”终止部署机器人 ★★★"
echo
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
echo
echo -e "\033[1;32m ===== <<开始部署gdutils查询转存TG机器人>> ===== \033[0m"
echo
#输入“机器人token/TG账号ID/域名/转存目的盘ID”
read -p """请输入机器人token并回车
Your Bot Token =>:""" YOUR_BOT_TOKEN
#判断token是否输入正确
while [[ "${#YOUR_BOT_TOKEN}" != 46 ]]; do
echo -e "\033[1;32m★★★ 机器人TOKEN输入不正确请重新输入或按“Ctrl+C”结束安装 ★★★\033[0m"
read -p """请输入机器人token并回车
Your Bot Token =>:""" YOUR_BOT_TOKEN
done
read -p """请输入你的域名(在cloudflare上解析到你机器人所在VPS的域名格式bot.abc.com)并回车
Your Domain Name =>:""" YOUR_DOMAIN_NAME
#判断域名是否正确
while [[ "$YOUR_DOMAIN_NAME" =~ "http" ]]; do
echo -e "\033[1;32m★★★ “Your Domain Name”输入错误应该输入你在cloudflare上解析的域名且不包含“http”请重新输入或按“Ctrl+C”结束安装 ★★★\033[0m"
read -p """请输入你的域名(在cloudflare上解析到你机器人所在VPS的域名格式bot.abc.com)并回车
Your Domain Name =>:""" YOUR_DOMAIN_NAME
done
read -p """请输入使用机器人的telegram账号ID(获取ID机器人@userinfobot)并回车
Your Telegram ID =>:""" YOUR_TELEGRAM_ID
#判断telegram ID是否正确(通过判断是不是纯数字)
until [[ $YOUR_TELEGRAM_ID =~ ^-?[0-9]+$ ]]; do
echo -e "\033[1;32m★★★ 您的TG账号ID输入不正确请重新输入或按“Ctrl+C”结束安装 ★★★\033[0m"
read -p """请输入使用机器人的telegram账号ID(获取ID机器人@userinfobot)并回车
Your Telegram ID =>:""" YOUR_TELEGRAM_ID
done
read -p """请输入转存默认目的地团队盘ID(不指定转存目的地默认改地址脚本强制要求输入团队盘ID)并回车
Your Google Team Drive ID =>:""" YOUR_GOOGLE_TEAM_DRIVE_ID
#判断google team drive ID是否正确团队盘ID长度19位
while [[ "${#YOUR_GOOGLE_TEAM_DRIVE_ID}" != 19 ]]; do
echo -e "\033[1;32m★★★ 您的Google team drive ID输入不正确请重新输入或按“Ctrl+C”结束安装 ★★★\033[0m"
read -p """请输入转存默认目的地ID(不指定转存目的地默认该地址脚本强制要求输入团队盘ID)并回车
Your Google Team Drive ID =>:""" YOUR_GOOGLE_TEAM_DRIVE_ID
done
cd ~ &&
sed -i "s/bot_token/$YOUR_BOT_TOKEN/g" ./gd-utils-cht/config.js &&
sed -i "s/your_tg_username/$YOUR_TELEGRAM_ID/g" ./gd-utils-cht/config.js &&
sed -i "s/DEFAULT_TARGET = ''/DEFAULT_TARGET = '$YOUR_GOOGLE_TEAM_DRIVE_ID'/g" ./gd-utils-cht/config.js
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
echo -e "\033[1;32m“进程守护程序pm2”开始安装......\033[0m"
cd /root/gd-utils-cht &&
npm i pm2 -g && pm2 l
echo -e "\033[1;32m启动守护进程......\033[0m"
pm2 start server.js --node-args="--max-old-space-size=4096"
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
echo -e "\033[1;32m“nginx”开始安装......\033[0m"
cd ~ &&
$cmd_install install nginx -y
echo
echo -e "\033[1;32m===== <<配置nginx服务>> ===== \033[0m"
echo
echo -e "\033[1;32m“nginx”起一个web服务......\033[0m"
cd $nginx_conf
echo "server {
listen 80;
server_name $YOUR_DOMAIN_NAME;
location / {
proxy_pass http://127.0.0.1:23333/;
}
}" >${nginx_conf}gdutilsbot.conf &&
$rm_nginx_default
ls &&
nginx -t &&
nginx -c /etc/nginx/nginx.conf &&
nginx -s reload &&
netstat -tulpen
echo -e "\033[1;32m----------------------------------------------------------\033[0m"
echo -e "\033[1;32m“检查网站是否部署成功”......\033[0m"
curl $YOUR_DOMAIN_NAME/api/gdurl/count\?fid=124pjM5LggSuwI1n40bcD5tQ13wS0M6wg
echo
echo -e "\033[1;32m设置Webhook服务......\033[0m"
print_webhook=$(curl -F "url=https://$YOUR_DOMAIN_NAME/api/gdurl/tgbot" "https://api.telegram.org/bot$YOUR_BOT_TOKEN/setWebhook")
echo
# 判断反向代理是否部署成功
if [[ $print_webhook =~ "true" ]]; then
echo -e "\033[1;32m★★★ 恭喜你GoogleDrive查询转存机器人部署成功请回到TG界面给bot发送个“/help”获取使用帮助 ★★★\033[0m"
else
echo -e "\033[32m★★★ 很遗憾!机器人设置失败,请返回检查网站是否部署成功,并重复本安装过程 ★★★\033[0m", exit!
fi
nginx -t && nginx -s reload
echo
echo
cd ~
rm -f gdutilsinstall.sh
###########################gdutils功能建议##################################
# 本部分是对gdutils项目的建议因为我主要用的是查询功能所以以下建议只涉及查询功能
# 1-把以下参数放入配置文件设置sa存放路径
# 2-改sa“随机”使用为“顺序”分组使用
# 3-增加输出模式,可以用命令行后带参数选择,具体模式建议:
# ①按一级或者二级文件夹显示数量大小
# ②可以一次性统计多个磁盘并且输出单个磁盘文件数和大小以及几个磁盘总和
# ③获取id对应的文件夹名或者磁盘明保存数据库给个命令能够查询历史记录汇总或者指定汇总
# 4-查询过程中输出方式不要每次都输出一次,可以固定+数字变化
# 5-命令参数可加在ID前或后如果非要固定一种的话就加在ID之前
# 6-命令行也改为默认sa模式
############################################################################

View File

@ -3,11 +3,34 @@
> 不只是最快的 google drive 拷貝工具 [與其他工具的對比](./compare.md)
> 我的readme可能不夠完全, 主要寫上我更新、修改的內容, 具體說明還是看[這邊](https://github.com/iwestlin/gd-utils)和[這邊](https://github.com/vitaminx/gd-utils)吧
## 從其他專案轉移至本繁中專案
```sh
pm2 delete 0
mv ./gd-utils/sa ./gd-utils/config.js ./gd-utils/gdurl.sqlite ./
rm -rf gd-utils
git clone https://github.com/liaojack8/gd-utils-cht && cd gd-utils-cht
npm install
rm -rf sa config.js gdurl.sqlite
cd ..
mv sa config.js gdurl.sqlite ./gd-utils-cht/
pm2 start ./gd-utils-cht/server.js
sudo pm2 save
```
- Demo Video: ~~[https://drive.google.com/file/d/1CltOaBDa4FVQ6doBP3S84MFPpbs2tv88](https://drive.google.com/file/d/1CltOaBDa4FVQ6doBP3S84MFPpbs2tv88)~~ (My account has been blocked)
## 更新紀錄
具體功能參考[iwestlin-changelog](https://github.com/iwestlin/gd-utils/blob/master/changelog.md)前期工作基本做完之後大概就是搬運了可能考慮做一下i18n
### 2020.07.30
- 同步原作者之更新 (清除按鈕、aria2.js等)
- 依舊保留了config_mod.js的修改項目, button-level的部分也套用清除按鈕的規則
### 2020.07.16
- 新增了從其他版本轉移到本專案的方式及教學
- 改用config_mod.js, 可自訂按鈕顯示的個數(每列), 可設定為1或2, 或自訂sa文件路徑(此文件不修改也可以正常使用bot)
### 2020.07.15
- 參照原作者更新, 加入單檔複製、tree列表的功能
### 2020.07.07
- 參照原作者@iwestlin更新tg.js及gd.js
- 整體繁體化, 介面部分
- 新增用戶可以在config.js自訂按鈕顯示的個數(每列), 可設定為1或2
- ~~新增用戶可以在config.js自訂按鈕顯示的個數(每列), 可設定為1或2~~
### 2020.07.06
- 部分繁體中文化
- 執行/task命令時, 會回傳完成度百分比
@ -30,15 +53,15 @@
- gdutils項目一鍵部署腳本包括“查詢轉存”和“TG機器人”兩部分
```
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilsinstall.sh)"
```
```
- gdutils項目一鍵部署腳本之“轉存查詢部分”
```
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilscsinstall.sh)"
```
```
- gdutils項目一鍵部署腳本之“TG機器人部分”
```
bash -c "$(curl -fsSL https://raw.githubusercontent.com/liaojack8/gd-utils-cht/master/gdutilsbotinstall.sh)"
```
```
- 安裝過程中需要輸入一下四個參數:
- 機器人TOKEN這個在Telegram裡面找“@BotFather”註冊即可獲得
- Telegram用戶ID在Telegram裡面向機器人@userinfobot发送消息即可獲得

172
src/gd.js
View File

@ -4,28 +4,57 @@ const dayjs = require('dayjs')
const prompts = require('prompts')
const pLimit = require('p-limit')
const axios = require('@viegg/axios')
const HttpsProxyAgent = require('https-proxy-agent')
const { GoogleToken } = require('gtoken')
const handle_exit = require('signal-exit')
const { argv } = require('yargs')
const { AUTH, RETRY_LIMIT, PARALLEL_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET, SA_PATH } = require('../config')
let { PARALLEL_LIMIT } = require('../config')
PARALLEL_LIMIT = argv.l || argv.limit || PARALLEL_LIMIT
const { AUTH, RETRY_LIMIT, TIMEOUT_BASE, TIMEOUT_MAX, LOG_DELAY, PAGE_SIZE, DEFAULT_TARGET } = require('../config')
const { SA_PATH } = require('../config_mod')
const { db } = require('../db')
const { make_table, make_tg_table, make_html, summary } = require('./summary')
const { gen_tree_html } = require('./tree')
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製'
const FILE_EXCEED_MSG = '您的小組雲端硬碟文件數量已超過限制(40萬),停止複製,請將未完成的資料夾移到另一個小組雲端硬碟中,再執行一遍複製指令即可繼斷點續傳'
const FOLDER_TYPE = 'application/vnd.google-apps.folder'
const { https_proxy } = process.env
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
const sleep = ms => new Promise((resolve, reject) => setTimeout(resolve, ms))
const { https_proxy, http_proxy, all_proxy } = process.env
const proxy_url = https_proxy || http_proxy || all_proxy
let axins
if (proxy_url) {
console.log('使用代理:', proxy_url)
let ProxyAgent
try {
ProxyAgent = require('proxy-agent')
} catch (e) { // 没执行 npm i proxy-agent
ProxyAgent = require('https-proxy-agent')
}
axins = axios.create({ httpsAgent: new ProxyAgent(proxy_url) })
} else {
axins = axios.create({})
}
const SA_LOCATION = argv.sa || 'sa'
const SA_BATCH_SIZE = 1000
const SA_FILES = fs.readdirSync(path.join(__dirname, SA_PATH)).filter(v => v.endsWith('.json'))
SA_FILES.flag = 0
let SA_TOKENS = get_sa_batch()
setInterval(() => {
SA_FILES.flag = 0
SA_TOKENS = get_sa_batch()
}, 1000 * 3600 * 12)
if (is_pm2()) {
setInterval(() => {
SA_FILES.flag = 0
SA_TOKENS = get_sa_batch()
}, 1000 * 3600 * 12)
}
// https://github.com/Leelow/is-pm2/blob/master/index.js
function is_pm2 () {
return 'PM2_HOME' in process.env || 'PM2_JSON_PROCESSING' in process.env || 'PM2_CLI' in process.env
}
function get_sa_batch () {
const new_flag = SA_FILES.flag + SA_BATCH_SIZE
@ -33,7 +62,7 @@ function get_sa_batch () {
SA_FILES.flag = new_flag
return files.map(filename => {
const gtoken = new GoogleToken({
keyFile: path.join(__dirname, '../sa', filename),
keyFile: path.join(__dirname, SA_PATH, filename),
scope: ['https://www.googleapis.com/auth/drive']
})
return { gtoken, expires: 0 }
@ -47,6 +76,7 @@ handle_exit(() => {
db.prepare('update task set status=? where id=?').run('interrupt', v.id)
})
records.length && console.log(records.length, 'task interrupted')
db.close()
})
async function gen_count_body ({ fid, type, update, service_account }) {
@ -71,6 +101,8 @@ async function gen_count_body ({ fid, type, update, service_account }) {
return (typeof smy === 'string') ? smy : JSON.stringify(smy)
}
}
const file = await get_info_by_id(fid, service_account)
if (file && file.mimeType !== FOLDER_TYPE) return render_smy(summary([file]), type)
let info, smy
const record = db.prepare('SELECT * FROM gd WHERE fid = ?').get(fid)
@ -120,7 +152,9 @@ async function count ({ fid, update, sort, type, output, not_teamdrive, service_
function get_out_str ({ info, type, sort }) {
const smy = summary(info, sort)
let out_str
if (type === 'html') {
if (type === 'tree') {
out_str = gen_tree_html(info)
} else if (type === 'html') {
out_str = make_html(smy)
} else if (type === 'json') {
out_str = JSON.stringify(smy)
@ -164,7 +198,7 @@ function get_all_by_fid (fid) {
}
async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
const result = []
let result = []
const not_finished = []
const limit = pLimit(PARALLEL_LIMIT)
@ -193,7 +227,7 @@ async function walk_and_save ({ fid, not_teamdrive, update, service_account }) {
should_save && save_files_to_db(parent, files)
const folders = files.filter(v => v.mimeType === FOLDER_TYPE)
files.forEach(v => v.parent = parent)
result.push(...files)
result = result.concat(files)
return Promise.all(folders.map(v => recur(v.id)))
}
try {
@ -235,15 +269,21 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
params.pageSize = Math.min(PAGE_SIZE, 1000)
// const use_sa = (fid !== 'root') && (service_account || !not_teamdrive) // 不带参数默认使用sa
const use_sa = (fid !== 'root') && service_account
const headers = await gen_headers(use_sa)
// const headers = await gen_headers(use_sa)
// 对于直接子文件数超多的目录1ctMwpIaBg8S1lrZDxdynLXJpMsm5guAl可能还没列完access_token就过期了
// 由于需要nextPageToken才能获取下一页的数据所以无法用并行请求测试发现每次获取1000个文件的请求大多需要20秒以上才能完成
const gtoken = use_sa && (await get_sa_token()).gtoken
do {
if (pageToken) params.pageToken = pageToken
let url = 'https://www.googleapis.com/drive/v3/files'
url += '?' + params_to_query(params)
const payload = { headers, timeout: TIMEOUT_BASE }
let retry = 0
let data
const payload = { timeout: TIMEOUT_BASE }
while (!data && (retry < RETRY_LIMIT)) {
const access_token = gtoken ? (await gtoken.getToken()).access_token : (await get_access_token())
const headers = { authorization: 'Bearer ' + access_token }
payload.headers = headers
try {
data = (await axins.get(url, payload)).data
} catch (err) {
@ -258,6 +298,7 @@ async function ls_folder ({ fid, not_teamdrive, service_account }) {
return files
}
files = files.concat(data.files)
argv.sfl && console.log('files.length:', files.length)
pageToken = data.nextPageToken
} while (pageToken)
@ -301,7 +342,7 @@ async function get_sa_token () {
try {
return await real_get_sa_token(tk)
} catch (e) {
console.log(e)
console.warn('SA獲取access_token失敗', e.message)
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== tk.gtoken)
if (!SA_TOKENS.length) SA_TOKENS = get_sa_batch()
}
@ -364,9 +405,9 @@ async function create_folder (name, parent, use_sa, limit) {
throw new Error(err_message + ' 目錄名:' + name)
}
async function get_name_by_id (fid) {
async function get_name_by_id (fid, use_sa) {
try {
const { name } = await get_info_by_id(fid, true)
const { name } = await get_info_by_id(fid, use_sa)
return name
} catch (e) {
return fid
@ -379,7 +420,7 @@ async function get_info_by_id (fid, use_sa) {
includeItemsFromAllDrives: true,
supportsAllDrives: true,
corpora: 'allDrives',
fields: 'id,name'
fields: 'id, name, size, parents, mimeType'
}
url += '?' + params_to_query(params)
const headers = await gen_headers(use_sa)
@ -433,7 +474,7 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
const record = db.prepare('select * from task where source=? and target=?').get(source, target)
if (record) {
const copied = db.prepare('select fileid from copied where taskid=?').all(record.id).map(v => v.fileid)
const choice = is_server ? 'continue' : await user_choose()
const choice = (is_server || argv.yes) ? 'continue' : await user_choose()
if (choice === 'exit') {
return console.log('退出程序')
} else if (choice === 'continue') {
@ -474,7 +515,7 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
if (min_size) files = files.filter(v => v.size >= min_size)
const folders = arr.filter(v => v.mimeType === FOLDER_TYPE)
console.log('待複製的目錄數:', folders.length)
console.log('待複製的檔案數:', files.length)
console.log('待複製的檔案數:', files.length)
const mapping = await create_folders({
source,
folders,
@ -513,23 +554,65 @@ async function real_copy ({ source, target, name, min_size, update, dncnr, not_t
}
async function copy_files ({ files, mapping, service_account, root, task_id }) {
if (!files.length) return
console.log('\n開始複製文件總數', files.length)
const limit = pLimit(PARALLEL_LIMIT)
let count = 0
const loop = setInterval(() => {
const now = dayjs().format('HH:mm:ss')
const message = `${now} | 已複製的檔案數 ${count} | 網路請求 進行中${limit.activeCount}/排隊中${limit.pendingCount}`
const message = `${now} | 已複製的檔案數 ${count} | 排隊中檔案數 ${files.length}`
print_progress(message)
}, 1000)
return Promise.all(files.map(async file => {
let count = 0
let concurrency = 0
let err
do {
if (err) {
clearInterval(loop)
files = null
throw err
}
if (concurrency > PARALLEL_LIMIT) {
await sleep(100)
continue
}
const file = files.shift()
if (!file) {
await sleep(1000)
continue
}
concurrency++
const { id, parent } = file
const target = mapping[parent] || root
const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
if (new_file) {
count++
db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
}
})).finally(() => clearInterval(loop))
copy_file(id, target, service_account, null, task_id).then(new_file => {
if (new_file) {
count++
db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
}
}).catch(e => {
err = e
}).finally(() => {
concurrency--
})
} while (concurrency || files.length)
return clearInterval(loop)
// const limit = pLimit(PARALLEL_LIMIT)
// let count = 0
// const loop = setInterval(() => {
// const now = dayjs().format('HH:mm:ss')
// const {activeCount, pendingCount} = limit
// const message = `${now} | 已复制文件数 ${count} | 网络请求 进行中${activeCount}/排队中${pendingCount}`
// print_progress(message)
// }, 1000)
// return Promise.all(files.map(async file => {
// const { id, parent } = file
// const target = mapping[parent] || root
// const new_file = await limit(() => copy_file(id, target, service_account, limit, task_id))
// if (new_file) {
// count++
// db.prepare('INSERT INTO copied (taskid, fileid) VALUES (?, ?)').run(task_id, id)
// }
// })).finally(() => clearInterval(loop))
}
async function copy_file (id, parent, use_sa, limit, task_id) {
@ -558,7 +641,7 @@ async function copy_file (id, parent, use_sa, limit, task_id) {
if (message && message.toLowerCase().includes('file limit')) {
if (limit) limit.clearQueue()
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
throw new Error('您的小組雲端硬碟文件數已超限,停止複製')
throw new Error(FILE_EXCEED_MSG)
}
if (use_sa && message && message.toLowerCase().includes('rate limit')) {
SA_TOKENS = SA_TOKENS.filter(v => v.gtoken !== gtoken)
@ -671,6 +754,22 @@ async function confirm_dedupe ({ file_number, folder_number }) {
return answer.value
}
// 需要sa是源文件夹所在盘的manager
async function mv_file ({ fid, new_parent, service_account }) {
const file = await get_info_by_id(fid, service_account)
if (!file) return
const removeParents = file.parents[0]
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
const params = {
removeParents,
supportsAllDrives: true,
addParents: new_parent
}
url += '?' + params_to_query(params)
const headers = await gen_headers(service_account)
return axins.patch(url, {}, { headers })
}
// 将文件或文件夹移入回收站,需要 sa 为 content manager 权限及以上
async function trash_file ({ fid, service_account }) {
const url = `https://www.googleapis.com/drive/v3/files/${fid}?supportsAllDrives=true`
@ -694,7 +793,7 @@ async function rm_file ({ fid, service_account }) {
}
}
async function dedupe ({ fid, update, service_account }) {
async function dedupe ({ fid, update, service_account, yes }) {
let arr
if (!update) {
const info = get_all_by_fid(fid)
@ -707,7 +806,7 @@ async function dedupe ({ fid, update, service_account }) {
const dupes = find_dupe(arr)
const folder_number = dupes.filter(v => v.mimeType === FOLDER_TYPE).length
const file_number = dupes.length - folder_number
const choice = await confirm_dedupe({ file_number, folder_number })
const choice = yes || await confirm_dedupe({ file_number, folder_number })
if (choice === 'no') {
return console.log('退出程序')
} else if (!choice) {
@ -727,7 +826,8 @@ async function dedupe ({ fid, update, service_account }) {
file_count++
}
} catch (e) {
console.log('刪除失敗', e.message)
console.log('刪除失敗', v)
handle_error(e)
}
}))
return { file_count, folder_count }
@ -751,4 +851,4 @@ function print_progress (msg) {
}
}
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id }
module.exports = { ls_folder, count, validate_fid, copy, dedupe, copy_file, gen_count_body, real_copy, get_name_by_id, get_info_by_id, get_access_token, get_sa_token, walk_and_save }

View File

@ -2,7 +2,7 @@ const Router = require('@koa/router')
const { db } = require('../db')
const { validate_fid, gen_count_body } = require('./gd')
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark } = require('./tg')
const { send_count, send_help, send_choice, send_task_info, sm, extract_fid, extract_from_text, reply_cb_query, tg_copy, send_all_tasks, send_bm_help, get_target_by_alias, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task } = require('./tg')
const { AUTH, ROUTER_PASSKEY, TG_IPLIST } = require('../config')
const { tg_whitelist } = AUTH
@ -42,16 +42,17 @@ router.get('/api/gdurl/count', async ctx => {
router.post('/api/gdurl/tgbot', async ctx => {
const { body } = ctx.request
console.log('ctx.ip', ctx.ip) // 可以只允许tg服务器的ip
console.log('tg message:', body)
console.log('tg message:', JSON.stringify(body, null, ' '))
if (TG_IPLIST && !TG_IPLIST.includes(ctx.ip)) return ctx.body = 'invalid ip'
ctx.body = '' // 早点释放连接
const message = body.message || body.edited_message
const message_str = JSON.stringify(message)
const { callback_query } = body
if (callback_query) {
const { id, data } = callback_query
const { id, message, data } = callback_query
const chat_id = callback_query.from.id
const [action, fid, target] = data.split(' ')
const [action, fid, target] = data.split(' ').filter(v => v)
if (action === 'count') {
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍候' })
counting[fid] = true
@ -67,33 +68,45 @@ router.post('/api/gdurl/tgbot', async ctx => {
tg_copy({ fid, target: get_target_by_alias(target), chat_id }).then(task_id => {
task_id && sm({ chat_id, text: `開始複製任務ID: ${task_id} 可輸入 /task ${task_id} 查詢進度` })
}).finally(() => COPYING_FIDS[fid] = false)
} else if (action === 'update') {
if (counting[fid]) return sm({ chat_id, text: fid + ' 正在統計,請稍等片刻' })
counting[fid] = true
send_count({ fid, chat_id, update: true }).finally(() => {
delete counting[fid]
})
} else if (action === 'clear_button') {
const { message_id, text } = message || {}
if (message_id) sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
}
return reply_cb_query({ id, data }).catch(console.error)
}
const chat_id = message && message.chat && message.chat.id
const text = message && message.text && message.text.trim()
const text = (message && message.text && message.text.trim()) || ''
let username = message && message.from && message.from.username
username = username && String(username).toLowerCase()
let user_id = message && message.from && message.from.id
user_id = user_id && String(user_id).toLowerCase()
if (!chat_id || !text || !tg_whitelist.some(v => {
if (!chat_id || !tg_whitelist.some(v => {
v = String(v).toLowerCase()
return v === username || v === user_id
})) return console.warn('異常請求')
})) {
chat_id && sm({ chat_id, text: '您的使用者名稱或ID不在機器人的白名單中如果是您配置的機器人請先到config.js中配置自己的username' })
return console.warn('收到非白名單用戶的請求')
}
const fid = extract_fid(text) || extract_from_text(text)
const fid = extract_fid(text) || extract_from_text(text) || extract_from_text(message_str)
const no_fid_commands = ['/task', '/help', '/bm']
if (!no_fid_commands.some(cmd => text.startsWith(cmd)) && !validate_fid(fid)) {
return sm({ chat_id, text: '未辨識到分享ID' })
}
if (text.startsWith('/help')) return send_help(chat_id)
if (text.startsWith('/bm')) {
const [cmd, action, alias, target] = text.split(' ').map(v => v.trim())
const [cmd, action, alias, target] = text.split(' ').map(v => v.trim()).filter(v => v)
if (!action) return send_all_bookmarks(chat_id)
if (action === 'set') {
if (!alias || !target) return sm({ chat_id, text: '標籤名和dstID不能為空' })
if (alias.length > 24) return sm({ chat_id, text: '標籤名請勿超過24个英文字符' })
if (alias.length > 24) return sm({ chat_id, text: '標籤名不要超過24個英文字符長度' })
if (!validate_fid(target)) return sm({ chat_id, text: 'dstID格式錯誤' })
set_bookmark({ chat_id, alias, target })
} else if (action === 'unset') {
@ -115,7 +128,7 @@ router.post('/api/gdurl/tgbot', async ctx => {
delete counting[fid]
}
} else if (text.startsWith('/copy')) {
let target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim())[1]
let target = text.replace('/copy', '').replace(' -u', '').trim().split(' ').map(v => v.trim()).filter(v => v)[1]
target = get_target_by_alias(target) || target
if (target && !validate_fid(target)) return sm({ chat_id, text: `目標ID ${target} 格式不正確` })
const update = text.endsWith(' -u')
@ -126,6 +139,15 @@ router.post('/api/gdurl/tgbot', async ctx => {
let task_id = text.replace('/task', '').trim()
if (task_id === 'all') {
return send_all_tasks(chat_id)
} else if (task_id === 'clear') {
return clear_tasks(chat_id)
} else if (task_id === '-h') {
return send_task_help(chat_id)
} else if (task_id.startsWith('rm')) {
task_id = task_id.replace('rm', '')
task_id = parseInt(task_id)
if (!task_id) return send_task_help(chat_id)
return rm_task({ task_id, chat_id })
}
task_id = parseInt(task_id)
if (!task_id) {
@ -134,10 +156,10 @@ router.post('/api/gdurl/tgbot', async ctx => {
return running_tasks.forEach(v => send_task_info({ chat_id, task_id: v.id }).catch(console.error))
}
send_task_info({ task_id, chat_id }).catch(console.error)
} else if (text.includes('drive.google.com/') || validate_fid(text)) {
return send_choice({ fid: fid || text, chat_id }).catch(console.error)
} else if (message_str.includes('drive.google.com/') || validate_fid(text)) {
return send_choice({ fid: fid || text, chat_id })
} else {
sm({ chat_id, text: '暫不支此命令' })
sm({ chat_id, text: '暫不支此命令' })
}
})

View File

@ -123,4 +123,4 @@ function format_size (n) {
flag++
}
return n.toFixed(2) + ' ' + units[flag]
}
}

138
src/tg.js
View File

@ -4,12 +4,13 @@ const axios = require('@viegg/axios')
const HttpsProxyAgent = require('https-proxy-agent')
const { db } = require('../db')
const { gen_count_body, validate_fid, real_copy, get_name_by_id } = require('./gd')
const { gen_count_body, validate_fid, real_copy, get_name_by_id, get_info_by_id, copy_file } = require('./gd')
const { AUTH, DEFAULT_TARGET, USE_PERSONAL_AUTH } = require('../config')
const { BUTTON_LEVEL } = require('../config_mod')
const { tg_token } = AUTH
const gen_link = (fid, text) => `<a href="https://drive.google.com/drive/folders/${fid}">${text || fid}</a>`
if (!tg_token) throw new Error('請先在config.js設定tg_token')
if (!tg_token) throw new Error('請先在config.js設定tg_token')
const { https_proxy } = process.env
const axins = axios.create(https_proxy ? { httpsAgent: new HttpsProxyAgent(https_proxy) } : {})
@ -18,13 +19,12 @@ const FID_TO_NAME = {}
async function get_folder_name (fid) {
let name = FID_TO_NAME[fid]
if (name) return name
name = await get_name_by_id(fid)
name = await get_name_by_id(fid, !USE_PERSONAL_AUTH)
return FID_TO_NAME[fid] = name
}
function send_help (chat_id) {
const text = `<pre>[使用說明]
***不支持單檔分享***
命令 說明
=====================
/help | 使
@ -39,6 +39,11 @@ sourceID可以是共享網址本身也可以是共享ID。如果命令最后
=====================
/task taskID() | taskID
若填 all 則返回所有任務列表(歷史紀錄)
/task |
/task 7 | ID 7
/task all |
/task clear | finished
/task rm 7 | 7
=====================
/bm [action] [alias] [target] | bookmarkID
會在輸入共享連結後返回的文件統計開始複製這兩個按鈕的下方出現方便複製到常用位置
@ -61,9 +66,35 @@ function send_bm_help (chat_id) {
return sm({ chat_id, text, parse_mode: 'HTML' })
}
function send_task_help (chat_id) {
const text = `<pre>/task [action/id] [id] | 查詢或管理任務進度
範例
/task |
/task 7 | ID 7
/task all |
/task clear | finished
/task rm 7 | 7
</pre>`
return sm({ chat_id, text, parse_mode: 'HTML' })
}
function clear_tasks (chat_id) {
const finished_tasks = db.prepare('select id from task where status=?').all('finished')
finished_tasks.forEach(task => rm_task({ task_id: task.id }))
sm({ chat_id, text: '已清除所有狀態為finished的任務紀錄' })
}
function rm_task ({ task_id, chat_id }) {
const exist = db.prepare('select id from task where id=?').get(task_id)
if (!exist) return sm({ chat_id, text: `不存在任務ID為 ${task_id} 的任務記錄` })
db.prepare('delete from task where id=?').run(task_id)
db.prepare('delete from copied where taskid=?').run(task_id)
if (chat_id) sm({ chat_id, text: `已刪除任務 ${task_id} 記錄` })
}
function send_all_bookmarks (chat_id) {
let records = db.prepare('select alias, target from bookmark').all()
if (!records.length) return sm({ chat_id, text: '資料庫中沒有收藏紀錄' })
if (!records.length) return sm({ chat_id, text: '數據庫中沒有收藏記錄' })
const tb = new Table({ style: { head: [], border: [] } })
const headers = ['標籤名', 'dstID']
records = records.map(v => [v.alias, v.target])
@ -76,7 +107,7 @@ function set_bookmark ({ chat_id, alias, target }) {
const record = db.prepare('select alias from bookmark where alias=?').get(alias)
if (record) return sm({ chat_id, text: '資料庫中已有同名的收藏' })
db.prepare('INSERT INTO bookmark (alias, target) VALUES (?, ?)').run(alias, target)
return sm({ chat_id, text: `成功設定收藏${alias} | ${target}` })
return sm({ chat_id, text: `成功設定收藏${alias} | ${target}` })
}
function unset_bookmark ({ chat_id, alias }) {
@ -91,6 +122,11 @@ function get_target_by_alias (alias) {
return record && record.target
}
function get_alias_by_target (target) {
const record = db.prepare('select alias from bookmark where target=?').get(target)
return record && record.alias
}
function send_choice ({ fid, chat_id }) {
if(BUTTON_LEVEL == 1){
return sm({
@ -104,7 +140,10 @@ function send_choice ({ fid, chat_id }) {
[
{ text: '開始複製', callback_data: `copy ${fid}` }
]
].concat(gen_bookmark_choices(fid))
].concat(gen_bookmark_choices(fid)).concat([[
{ text: '強制更新', callback_data: `update ${fid}` },
{ text: '清除按鈕', callback_data: `clear_button` }
]])
}
})
}else{
@ -117,7 +156,10 @@ function send_choice ({ fid, chat_id }) {
{ text: '文件統計', callback_data: `count ${fid}` },
{ text: '開始複製', callback_data: `copy ${fid}` }
]
].concat(gen_bookmark_choices(fid))
].concat(gen_bookmark_choices(fid)).concat([[
{ text: '強制更新', callback_data: `update ${fid}` },
{ text: '清除按鈕', callback_data: `clear_button` }
]])
}
})
}
@ -131,16 +173,12 @@ function gen_bookmark_choices (fid) {
}else{
level = BUTTON_LEVEL
}
const gen_choice = v => ({text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}`})
const gen_choice = v => ({ text: `複製到 ${v.alias}`, callback_data: `copy ${fid} ${v.alias}` })
const records = db.prepare('select * from bookmark').all()
db.close()
const result = []
for (let i = 0; i < records.length; i++) {
for (let i = 0; i < records.length; i += 2) {
const line = [gen_choice(records[i])]
for(let j = 0; j < level-1; j ++){
if (records[i+1]) line.push(gen_choice(records[i+1]))
i++
}
if (records[i + 1]) line.push(gen_choice(records[i + 1]))
result.push(line)
}
return result
@ -166,8 +204,8 @@ async function send_all_tasks (chat_id) {
// const description = err.response && err.response.data && err.response.data.description
// if (description && description.includes('message is too long')) {
if (true) {
const text = [headers].concat(records).map(v => v.join('\t')).join('\n')
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務\n<pre>${text}</pre>` })
const text = [headers].concat(records.slice(-100)).map(v => v.join('\t')).join('\n')
return sm({ chat_id, parse_mode: 'HTML', text: `所有拷貝任務(僅顯示最近100項)\n<pre>${text}</pre>` })
}
console.error(err)
})
@ -176,30 +214,26 @@ async function send_all_tasks (chat_id) {
async function get_task_info (task_id) {
const record = db.prepare('select * from task where id=?').get(task_id)
if (!record) return {}
const { source, target, status, copied, mapping, ctime, ftime } = record
const { source, target, status, mapping, ctime, ftime } = record
const { copied_files } = db.prepare('select count(fileid) as copied_files from copied where taskid=?').get(task_id)
const folder_mapping = mapping && mapping.trim().split('\n')
const new_folder = folder_mapping && folder_mapping[0].split(' ')[1]
const { summary } = db.prepare('select summary from gd where fid=?').get(source) || {}
const { file_count, folder_count, total_size } = summary ? JSON.parse(summary) : {}
const copied_files = copied ? copied.trim().split('\n').length : 0
const total_count = (file_count || 0) + (folder_count || 0)
const copied_folders = folder_mapping ? (folder_mapping.length - 1) : 0
let text = '任務ID' + task_id + '\n'
let text = '任務ID' + task_id + ' [' + status + ']\n'
const folder_name = await get_folder_name(source)
text += '源資料夾:' + gen_link(source, folder_name) + '\n'
text += '目的位置:' + gen_link(target) + '\n'
text += '目的位置:' + gen_link(target, get_alias_by_target(target)) + '\n'
text += '新資料夾:' + (new_folder ? gen_link(new_folder) : '尚未創建') + '\n'
text += '任務狀態:' + status + '\n'
text += '創建時間:' + dayjs(ctime).format('YYYY-MM-DD HH:mm:ss') + '\n'
text += '完成時間:' + (ftime ? dayjs(ftime).format('YYYY-MM-DD HH:mm:ss') : '未完成') + '\n'
var pct = copied_folders/(folder_count === undefined ? '未知數量' : folder_count)*100
pct = pct.toFixed(2);
text += '目錄進度:' + copied_folders + '/' + (folder_count === undefined ? '未知數量' : folder_count) + ' - ' + pct + '%\n'
pct = copied_files/(file_count === undefined ? '未知數量' : file_count)*100
pct = pct.toFixed(2);
text += '文件進度:' + copied_files + '/' + (file_count === undefined ? '未知數量' : file_count) + ' - ' + pct + '%\n'
text += '目錄進度:' + copied_folders + '/' + (folder_count === undefined ? '未知數量' : folder_count) + ' - ' + (copied_folders/folder_count*100).toFixed(3) + '%\n'
text += '文件進度:' + copied_files + '/' + (file_count === undefined ? '未知數量' : file_count) + ' - ' + (copied_files/file_count*100).toFixed(3) + '%\n'
text += '合計大小:' + (total_size || '未知大小')
const total_count = (folder_count || 0) + (file_count || 0)
return { text, status, total_count }
return { text, status, folder_count }
}
async function send_task_info ({ task_id, chat_id }) {
@ -213,13 +247,12 @@ async function send_task_info ({ task_id, chat_id }) {
} catch (e) {
console.log('fail to send message to tg', e.message)
}
// get_task_info 在task目录数超大时比较吃cpu如果超1万就不每10秒更新了以后如果把mapping 也另存一张表可以取消此限制
if (!message_id || status !== 'copying' || folder_count > 10000) return
// get_task_info 在task目录数超大时比较吃cpu以后如果最好把mapping也另存一张表
if (!message_id || status !== 'copying') return
const loop = setInterval(async () => {
const url = `https://api.telegram.org/bot${tg_token}/editMessageText`
const { text, status } = await get_task_info(task_id)
if (status !== 'copying') clearInterval(loop)
axins.post(url, { chat_id, message_id, text, parse_mode: 'HTML' }).catch(e => console.error(e.message))
sm({ chat_id, message_id, text, parse_mode: 'HTML' }, 'editMessageText')
}, 10 * 1000)
}
@ -229,6 +262,14 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
sm({ chat_id, text: '請輸入目的地ID或先在config.js中設定預設複製的目的地ID(DEFAULT_TARGET)' })
return
}
const file = await get_info_by_id(fid, !USE_PERSONAL_AUTH)
if (file && file.mimeType !== 'application/vnd.google-apps.folder') {
return copy_file(fid, target, !USE_PERSONAL_AUTH).then(data => {
sm({ chat_id, parse_mode: 'HTML', text: `單檔複製成功,位置:${gen_link(target)}` })
}).catch(e => {
sm({ chat_id, text: `單檔複製成功失敗,失敗訊息:${e.message}` })
})
}
let record = db.prepare('select id, status from task where source=? and target=?').get(fid, target)
if (record) {
@ -242,7 +283,7 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
real_copy({ source: fid, update, target, service_account: !USE_PERSONAL_AUTH, is_server: true })
.then(async info => {
if (!record) record = {} // 防止无限循环
if (!record) record = {} // 防止無限循環
if (!info) return
const { task_id } = info
const { text } = await get_task_info(task_id)
@ -250,14 +291,11 @@ async function tg_copy ({ fid, target, chat_id, update }) { // return task_id
})
.catch(err => {
const task_id = record && record.id
if (task_id){
db.prepare('update task set status=? where id=?').run('error', task_id)
db.close()
}
if (task_id) db.prepare('update task set status=? where id=?').run('error', task_id)
if (!record) record = {}
console.error('複製失敗', fid, '-->', target)
console.error(err)
sm({ chat_id, text: '複製失敗,失敗訊息:' + err.message })
sm({ chat_id, text: (task_id || '') + '複製失敗,失敗訊息:' + err.message })
})
while (!record) {
@ -310,18 +348,21 @@ ${table}</pre>`
文件總數${file_count}
目錄總數${folder_count}
合計大小${total_size}
</pre>`
</pre>`
})
}
throw err
})
}
function sm (data) {
const url = `https://api.telegram.org/bot${tg_token}/sendMessage`
function sm (data, endpoint) {
endpoint = endpoint || 'sendMessage'
const url = `https://api.telegram.org/bot${tg_token}/${endpoint}`
return axins.post(url, data).catch(err => {
// console.error('fail to post', url, data)
console.error('fail to send message to tg:', err.message)
const err_data = err.response && err.response.data
err_data && console.error(err_data)
})
}
@ -333,9 +374,11 @@ function extract_fid (text) {
if (!text.startsWith('http')) text = 'https://' + text
const u = new URL(text)
if (u.pathname.includes('/folders/')) {
const reg = /[^/?]+$/
const match = u.pathname.match(reg)
return match && match[0]
return u.pathname.split('/').map(v => v.trim()).filter(v => v).pop()
} else if (u.pathname.includes('/file/')) {
const file_reg = /file\/d\/([a-zA-Z0-9_-]+)/
const file_match = u.pathname.match(file_reg)
return file_match && file_match[1]
}
return u.searchParams.get('id')
} catch (e) {
@ -344,9 +387,10 @@ function extract_fid (text) {
}
function extract_from_text (text) {
const reg = /https?:\/\/drive.google.com\/[^\s]+/g
// const reg = /https?:\/\/drive.google.com\/[^\s]+/g
const reg = /https?:\/\/drive.google.com\/[a-zA-Z0-9_\\/?=&-]+/g
const m = text.match(reg)
return m && extract_fid(m[0])
}
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark }
module.exports = { send_count, send_help, sm, extract_fid, reply_cb_query, send_choice, send_task_info, send_all_tasks, tg_copy, extract_from_text, get_target_by_alias, send_bm_help, send_all_bookmarks, set_bookmark, unset_bookmark, clear_tasks, send_task_help, rm_task }

115
src/tree.js Normal file
View File

@ -0,0 +1,115 @@
module.exports = { gen_tree_html }
function gen_tree_html (arr) {
const data = gen_tree_data(arr, is_gd_folder)
return tree_tpl(JSON.stringify(data))
}
function tree_tpl (str) {
return `<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no">
<meta name="theme-color" content="#000000">
<title>Folder Tree</title>
<link href="https://cdn.jsdelivr.net/gh/iwestlin/gd-utils/static/tree.min.css" rel="stylesheet">
</head>
<body>
<!-- source code: https://github.com/iwestlin/foldertree/blob/master/app.jsx -->
<div id="root"></div>
<script type="text/javascript">
var treedata = ${str}
</script>
<script type="text/javascript" src="https://cdn.jsdelivr.net/gh/iwestlin/gd-utils/static/tree.min.js"></script>
</body>
</html>`
}
function is_gd_folder (data) {
return data.mimeType === 'application/vnd.google-apps.folder'
}
function gen_tree_data (data, is_folder) {
if (!data || !data.length) return []
const folders = data.filter(is_folder)
const files = data.filter(v => !is_folder(v))
const total_size = sum(files.map(v => v.size))
const root = {
title: `/根目录 [共${files.length} 个文件(不包括文件夹), ${format_size(total_size)}]`,
key: data[0].parent
}
if (!folders.length) return [root]
const sub_folders = folders.filter(v => v.parent === folders[0].parent)
sub_folders.forEach(v => {
sum_files(v, data, is_folder)
count_files(v, data, is_folder)
})
sort_folders(folders, 'count')
sort_folders(sub_folders, 'count')
folders.forEach(v => {
let {name, size, count, id} = v
if (name.length > 50) name = name.slice(0, 48) + '...'
v.title = `${name} | [共${count}个文件 ${format_size(size)}]`
})
root.children = sub_folders.map(v => gen_node(v, folders))
return [root]
}
function sort_folders (folders, type) {
if (!folders || !folders.length) return
if (type === 'size') return folders.sort((a, b) => b.size - a.size)
if (type === 'count') return folders.sort((a, b) => b.count - a.count)
}
function gen_node (v, folders) {
const {id, title, node} = v
if (node) return node
return v.node = {
title,
key: id,
children: v.children || folders.filter(vv => vv.parent === id).map(vv => gen_node(vv, folders))
}
}
function count_files (folder, arr, is_folder) {
if (folder.count) return folder.count
const children = arr.filter(v => v.parent === folder.id)
return folder.count = sum(children.map(v => {
if (is_folder(v)) return count_files(v, arr, is_folder)
return 1
}))
}
function sum_files (folder, arr, is_folder) {
if (folder.size) return folder.size
const children = arr.filter(v => v.parent === folder.id)
return folder.size = sum(children.map(v => {
if (is_folder(v)) return sum_files(v, arr, is_folder)
return v.size
}))
}
function sum (arr) {
let result = 0
for (const v of arr) {
result += Number(v) || 0
}
return result
}
function format_size (n) {
n = Number(n)
if (Number.isNaN(n)) return ''
if (n < 0) return 'invalid size'
const units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
let flag = 0
while (n >= 1024) {
n = n / 1024
flag++
}
return n.toFixed(2) + ' ' + units[flag]
}

BIN
static/colab.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 208 KiB

1
static/tree.min.css vendored Normal file

File diff suppressed because one or more lines are too long

1
static/tree.min.js vendored Normal file

File diff suppressed because one or more lines are too long

BIN
static/tree.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 176 KiB

View File

@ -1,7 +1,7 @@
#!/usr/bin/env node
const { argv } = require('yargs')
.usage('用法: ./$0 folder-id [options]\nfolder-id 是你想检测SA是否对其有阅读权限的目录ID')
.usage('用法: ./$0 folder-id\nfolder-id 是你想檢測SA是否對其有閱讀權限的目錄ID')
.help('h')
.alias('h', 'help')
@ -28,18 +28,18 @@ main()
async function main () {
const [fid] = argv._
if (validate_fid(fid)) {
console.log('开始检测', SA_TOKENS.length, '个SA帐号')
console.log('開始檢測', SA_TOKENS.length, '個SA帳號')
const invalid_sa = await get_invalid_sa(SA_TOKENS, fid)
if (!invalid_sa.length) return console.log('已检测', SA_TOKENS.length, '个SA未检测到无效帐号')
if (!invalid_sa.length) return console.log('已檢測', SA_TOKENS.length, '個SA未檢測到無效帳號')
const choice = await choose(invalid_sa.length)
if (choice === 'yes') {
mv_sa(invalid_sa)
console.log('成功移')
console.log('成功移')
} else {
console.log('成功退出,无效的SA记录', invalid_sa)
console.log('成功退出,無效的SA記錄', invalid_sa)
}
} else {
console.warn('目录ID缺失或格式错误')
console.warn('目錄ID缺失或格式錯誤')
}
}
@ -55,9 +55,9 @@ async function choose (count) {
const answer = await prompts({
type: 'select',
name: 'value',
message: `检测到 ${count} 个无效的SA是否将它们移动到 sa/invalid 目录下?`,
message: `檢測到 ${count} 個無效的SA是否將它們移動到 sa/invalid 目錄下?`,
choices: [
{ title: 'Yes', description: '确认移动', value: 'yes' },
{ title: 'Yes', description: '確認移動', value: 'yes' },
{ title: 'No', description: '不做更改,直接退出', value: 'no' }
],
initial: 0
@ -66,30 +66,40 @@ async function choose (count) {
}
async function get_invalid_sa (arr, fid) {
if (!fid) throw new Error('请指定要检测权限的目录ID')
if (!fid) throw new Error('請指定要檢測權限的目錄ID')
const fails = []
let flag = 0
let good = 0
for (const v of arr) {
console.log('检测进度', `${flag++}/${arr.length}`)
console.log('正常/常', `${good}/${fails.length}`)
console.log('檢測進度', `${flag++}/${arr.length}`)
console.log('正常/常', `${good}/${fails.length}`)
const {gtoken, filename} = v
try {
const access_token = await get_sa_token(gtoken)
await get_info(fid, access_token)
good++
} catch (e) {
handle_error(e)
const status = e && e.response && e.response.status
if (Number(status) === 400) fails.push(filename) // access_token 获取失败
const data = e && e.response && e.response.data
const code = data && data.error && data.error.code
if (Number(code) === 404) fails.push(filename) // 读取文件夹信息失败
if ([404, 403].includes(Number(code))) fails.push(filename) // 读取文件夹信息失败
}
}
return fails
}
function handle_error (err) {
const data = err && err.response && err.response.data
if (data) {
console.error(JSON.stringify(data))
} else {
console.error(err.message)
}
}
async function get_info (fid, access_token) {
let url = `https://www.googleapis.com/drive/v3/files/${fid}`
let params = {