当前位置 博文首页 > 吃西瓜的星星:利用github给国外文件下载加速

    吃西瓜的星星:利用github给国外文件下载加速

    作者:吃西瓜的星星 时间:2021-01-17 06:01

    前言

    作为一名程序员,经常需要下载一些编程相关的环境,而国内的网络环境大家都知道,有的文件用浏览器是下载不动的,于是我有了利用github下载文件的想法。

    我的demo项目地址:https://github.com/bobowire/wireboy.remote.download

    参考资料:

    1. NodeJS使用node-fetch下载文件并显示下载进度示例:https://www.jianshu.com/p/4b58711cb72a
    2. nodejs——发送邮件(带附件):https://www.cnblogs.com/yourstars/p/6728931.html

    觉得好玩的,大家可以点个赞~

    下载加速原理

    使用github的Action远程执行文件下载(下载qt环境速度可以达到3mb/s),然后将下载的文件进行分片,每片15mb,分片后的文件以邮件附件的方式发送到国内邮箱,我们通过下载邮箱中的附件,将分片的附件合并成完整文件,从而实现不翻墙、不用下载器也能下载国外文件的目的。

    简单点说

    1. github远程下载
    2. 文件分片
    3. 通过邮箱发到国内
    4. 对附件进行合并

    使用方法

    1. 新建github项目
    2. 创建js文件download.js文件,内容请查阅后文
    3. 创建workflows/RunDownload.yml文件,内容请查阅后文
    4. 修改download.js中的fileURL 变量值,此为文件url地址
    5. 在项目github->settings->Secrets中,点击右上方“new responsitory secret”按钮,添加"EMAILPASS","SENDEMAIL","TOEMAIL"变量(授权码、发送邮箱、目标邮箱)
    6. 以上全部完成后,我们每次修改download.js文件的fileURL地址,github都会自动进行一次下载。原理请自行百度“github action”。

    注意:

    1. 授权码(EMAILPASS)是指“邮箱第三方登录授权码”,如何获取授权码,以QQ邮箱为例,请点击:http://jingyan.baidu.com/article/fedf0737af2b4035ac8977ea.html

    github Action文件(RunDownload.yml)

    name: Github wireboy.remote.download
     
    on:
        push:
            branches:
                - main
        schedule:
         - cron: '* * * * *'
    jobs:
        build:
            runs-on: ubuntu-latest
     
            steps:
                - name: Checkout codes
                  uses: actions/checkout@v2
                - name: Use Node.js
                  uses: actions/setup-node@v1
                  with:
                    node-version: '12.x'
                - name: Run
                  run: npm install
                  
                - run: node download.js
                  
                  env:
                    EMAILPASS: ${{ secrets.EMAILPASS }}
                    SENDEMAIL: ${{ secrets.SENDEMAIL }}
                    TOEMAIL: ${{ secrets.TOEMAIL }}
    
    

    源代码(download.js)

    const fetch = require("node-fetch");
    const fs = require("fs");
    const path = require("path");
    const progressStream = require('progress-stream');
    const nodemailer = require('nodemailer');
    
    
    //下载 的文件 地址 (https://nodejs.org/dist/v12.18.3/node-v12.18.3-x64.msi)
    let fileURL = 'https://nodejs.org/dist/v12.18.3/node-v12.18.3-x64.msi';
    
    
    //下载保存的文件路径
    let fileSavePath = path.join(__dirname, path.basename(fileURL));
    //缓存文件路径
    let tmpFileSavePath = fileSavePath + ".tmp";
    //创建写入流
    const fileStream = fs.createWriteStream(tmpFileSavePath).on('error', function (e) {
        console.error('error==>', e)
    }).on('ready', function () {
        console.log("开始下载:", fileURL);
    }).on('finish', function () {
        //下载完成后重命名文件
        fs.renameSync(tmpFileSavePath, fileSavePath);
        console.log('文件下载完成:', fileSavePath);
        const readstream = fs.createReadStream(fileSavePath);
    	let i = 0;
    	console.time('readtime');
        let patchIndex = 0;
    	readstream.on('readable', () => {
            {
                console.log('start read');
                let chunk = readstream.read(1024 * 1024 * 15);
                while (null !== chunk) {
                    patchIndex = patchIndex + 1;
                    console.log('read times:'+patchIndex)
                    console.log(fileSavePath+'.email_'+patchIndex);
                    let emailFile = fs.createWriteStream(fileSavePath+'.email_'+patchIndex).on('finish',function(){
        
                    })
                    emailFile.write(chunk);
                    emailFile.end();
                    let msg = createEmailMessage(patchIndex+'_'+path.basename(fileURL),fileSavePath+'.email_'+patchIndex,path.basename(fileURL) + '(' + patchIndex + ')');
                    console.log('Send Mail ' + patchIndex + ' times');
                    console.log(path.basename(fileURL));
                    var transporter = createTransporter();
                    transporter.sendMail(msg, (error, info) => {
                        if (error) {
                            console.log('Error occurred');
                            console.log(error.message);
                            return;
                        }
                        console.log('Message sent successfully!');
                        console.log('Server responded with "%s"', info.response);
                        transporter.close();
                    });
        
                    chunk = readstream.read(1024 * 1024 * 10);
                }
                console.log('end read');
            }
    	});
    	readstream.on('close', () => {
    		console.timeEnd('readtime');
        });
        
    });
    //请求文件
    fetch(fileURL, {
        method: 'GET',
        headers: { 'Content-Type': 'application/octet-stream' },
        // timeout: 100,    
    }).then(res => {
        //获取请求头中的文件大小数据
        let fsize = res.headers.get("content-length");
        //创建进度
        let str = progressStream({
            length: fsize,
            time: 100 /* ms */
        });
        // 下载进度 
        str.on('progress', function (progressData) {
            //不换行输出
            let percentage = Math.round(progressData.percentage) + '%';
            console.log(percentage);
            // process.stdout.write('\033[2J'+);
            // console.log(progress);
            /*
            {
                percentage: 9.05,
                transferred: 949624,
                length: 10485760,
                remaining: 9536136,
                eta: 42,
                runtime: 3,
                delta: 295396,
                speed: 949624
            }
            */
        });
        res.body.pipe(str).pipe(fileStream);
    }).catch(e => {
        //自定义异常处理
        console.log(e);
    });
    
    
    
    var createTransporter = function(){
        return nodemailer.createTransport({
            service: 'qq',
            auth: {
                user: process.env.SENDEMAIL,//发送者邮箱
                pass: process.env.EMAILPASS //邮箱第三方登录授权码
            },
           //  logger: bunyan.createLogger({
           //      name: 'nodemailer'
           //  }),//打印日志
            debug: true
        },{
            from: process.env.SENDEMAIL,//发送者邮箱
            headers: {
                'X-Laziness-level': 1000
            }
        });
    } 
    
    console.log('SMTP Configured');
    
    var createEmailMessage = function(filename,filepath,subject){
        var message = {
            // Comma separated lsit of recipients 收件人用逗号间隔
            to: process.env.TOEMAIL,
        
            // Subject of the message 信息主题
            subject:  subject,
        
            // plaintext body
            text: '请查阅附件',
        
            // Html body
            html: '<p>下载文件成功</p>',
        
            // Apple Watch specific HTML body 苹果手表指定HTML格式
            watchHtml: '<b>Hello</b> to myself',
        
            // An array of attachments 附件
            attachments: [
                // String attachment
               //  {
               //      filename: 'notes.txt',
               //      content: 'Some notes about this e-mail',
               //      contentType: 'text/plain' // optional,would be detected from the filename 可选的,会检测文件名
               //  },
               //  // Binary Buffer attchment
               //  {
               //      filename: 'image.png',
               //      content: Buffer.from('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' +
               //         '//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' +
               //         'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'),
               //      cid: '00001'  // should be as unique as possible 尽可能唯一
               //  },
                // File Stream attachment
                {
                    filename: filename,
                    path: filepath,
                    // cid: '00002'  // should be as unique as possible 尽可能唯一
                 }
            ]
        
        };
        return message;
    };
    
    

    效果图

    收件箱

    github的action运行日志

    下载VisualCode运行图