zoukankan      html  css  js  c++  java
  • 浏览器整站下载工具

     1、下载安装nodejs,

    2、npm安装模块 puppeteer-core,

    3、然后运行当前js,参考demo输入url,就会下载所有的css、js、html等

    //整站下载工具
    const puppeteer = require('puppeteer-core');
    const path = require('path');
    //mkdir.js
    const fs=require('fs');
    const dirCache={};
    function mkdir(filepath) {
        const arr=filepath.split('/');
        let dir=arr[0];
        for(let i=1;i<arr.length;i++){
            if(!dirCache[dir]&&!fs.existsSync(dir)){
                dirCache[dir]=true;
                fs.mkdirSync(dir);
            }
            dir=dir+'/'+arr[i];
        }
    }//获取chrome浏览器第地址
    const getDefaultOsPath = () => {
        if (process.platform === 'win32') {
            return 'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'
        } else {
            return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
        }
    };
    const getUserDataDir = () => {
        if (process.platform === 'win32') {
            return 'D:\User Data'
        } else {
            return '/Users/caoke/chromeUserdata'
        }
    }
    //睡眠
    const sleep=function (time) {
        return new Promise(function (resolve) {
            setTimeout(resolve,time)
        })
    };
    //语法糖
    async function runActions(arr){
        const page=this;
        for(let i=0;i<arr.length;i++){
            const action=arr[i];
            const key=action[0];
            if(key==='click'){
                page.click(action[1]);
            }else if(key==='keypress'){
                await page.keyboard.type(action[1]);
            }else if(key==='clickAndtype'){
                await page.type(action[1],action[2],{delay: 100});
            }else if(key==='type'){
                await page.keyboard.type(action[1],{delay: 100});
            }else if(key==='sleep'){
                await sleep(action[1])
            }else if(key==='uploadFile'){
                const elementHandle=await page.$(action[1]);
                await elementHandle.uploadFile(action[2])
            }else if(key==='goBack'){
                await page.goBack()
            }else  if(key==='userAgent'){
                await page.setUserAgent(action[1])
            }else if(key==='viewport'){
                await page.setViewport(action[1])
            }else if(key==='openUrl'){
                await page.goto(action[1]);
            }else if(key==='emulate'){
                await page.emulate(puppeteer.devices[action[1]]);
            }
        }
    
    
    };
    async function loadStatic(astionArr) {
        const browser = await puppeteer.launch({
            // headless: false,
            // ignoreDefaultArgs:true,
            // args :['--user-data-dir='+getUserDataDir()],
            executablePath:getDefaultOsPath(),
        });
        const page=await browser.newPage();
        page.on('response',async function (res) {
            if(!res){return;}
            try {
                const buffer=await res.buffer();
                if(!res.fromCache()&&buffer){
                    const url=res.url();
                    let filepath='./dist/'+url.replace(':/','').replace(/:/g,'').replace(/?.+$/,'');
                    mkdir(filepath);
                    fs.writeFileSync(filepath,buffer);
                }
            }
            catch (e) {}
        })
        page.runActions=runActions;
        await page.runActions(astionArr);
        await browser.close();
    }
    module.exports=loadStatic;
    
    //demo
    loadStatic([
    ['openUrl','https://zhangyaochun.iteye.com/blog/2029910'],
    ['sleep',3000],
    ['openUrl','https://www.baidu.com'],
    ['sleep',3000],
    ])

    下载nodejs,npm安装模块 puppeteer-core,然后运行当前js,参考demo输入url,就会下载所有的css、js、html等

  • 相关阅读:
    【Spring实战】—— 16 基于JDBC持久化的事务管理
    【Spring实战】—— 15 Spring JDBC模板使用
    (转)Spring Boot(九):定时任务
    (转)Spring Boot(八):RabbitMQ 详解
    (转)Spring Boot(七):Mybatis 多数据源最简解决方案
    (转)Spring Boot(六):如何优雅的使用 Mybatis
    (转)Spring Boot(五):Spring Boot Jpa 的使用
    (转)Spring Boot(四):Thymeleaf 使用详解
    (转)Spring Boot(三):Spring Boot 中 Redis 的使用
    (转)Spring Boot(二):Web 综合开发
  • 原文地址:https://www.cnblogs.com/caoke/p/11244916.html
Copyright © 2011-2022 走看看