Node脚本快速同步CNPM项目内用到的依赖

Posted 2021-05-18 crper

前言

还是为了解决之前的问题;
公司用CNPM作为内部私有仓,没有开启全量实时同步;
所以有些包会相对落后,所以常用同步上游就显得很重要了;


我想了想,每次都要手动去执行个别的包或者少量包的查询,操作太多了;
原理还是遵循CNPM更新机制,可以看看上篇帖子哈~

考虑的点

• 设置一个根路径,会自动检索下所有项目的packaeg.json(不包含node_modules)
  • 包括所有git subtree或者monorepo的package.json
• 支持延时执行,一瞬间太多要同步的,会把内部搭建cnpm搞崩;
• 同步过,再下一个执行同步的会自动过滤.也就是同步过同名包不会再发同步请求

使用成本极低,一个Node环境装几个常用的npm包;

环境

• Node 14.16.1

效果图

源码

const globby = require('globby');
const fs = require('fs');
const path = require('path');
const axios = require('axios');
const chalk = require('chalk');
const isPlainObject = require('lodash/isPlainObject');
const options = {
    baseRootPath: '/Users/linqunhe/Code',  // 检索的根路径
    ignorePackage: ['@ones-ai', '@ones'], // 忽略的包名,就是不管有木有缓存都不同步
    delayTime: 100, // 每一次执行延时的时间,随着执行次数会递增 , 2000 = 2s
    maxRetry: 3, // 整个逻辑,中间有错误重试机制最大次数
    privateHostName: 'https://xxxx.xxxx.xxxx', // 内网部署CNPM的访问域名
}
let cachePkgList = [];
let retryCount = 0;

function onesNpmSyncUpdate(pkgList, isArray = false) {
    const pkg = isArray ? pkgList.join(',') : pkgList
    return axios.put(`${options.privateHostName}/sync/${pkg}?sync_upstream=true`).then(res => {
        if (res && res.data && res.data.ok) {
            const data = [
                {
                    '执行时间': new Date().toISOString(),
                    'NPM包名': isArray ? JSON.stringify(pkgList) : pkgList,
                    '同步状态': res.data.ok
                }
            ]
            console.dir(data);
        }
    }).catch(err => {
        if (err) console.log('🍑 NPM包名', chalk.red(`${pkg}`.padEnd(60)), '👀 同步状态:  ', chalk.green('false'));
    })
}

function arrayTypeData(array) {
    let decoratorsArr = []
    let normalArr = []
    for (let item of array) {
        if (item && typeof item === 'string') {
            if (item.startsWith('@') && item.includes('/')) {
                decoratorsArr.push(item)
            } else {
                normalArr.push(item)
            }
        }
    }
    return {
        decoratorsArr,
        normalArr
    }
}

function getPackageJsonDepKey(json = { dependencies: {}, devDependencies: {} }, ignore = []) {
    const { dependencies, devDependencies, peerDependencies } = json;
    let dependenciesKey = [];
    let devDependenciesKey = [];
    let peerDependenciesKey = [];
    if (dependencies && isPlainObject(dependencies)) {
        dependenciesKey = Object.keys(dependencies);
    }

    if (devDependencies && isPlainObject(devDependencies)) {
        devDependenciesKey = Object.keys(devDependencies);
    }
    if (peerDependencies && isPlainObject(peerDependencies)) {
        peerDependenciesKey = Object.keys(peerDependencies);
    }

    const allDepKey = [...new Set([...dependenciesKey, ...devDependenciesKey, ...peerDependenciesKey])]
    return allDepKey.filter(item => {
        for (const iterator of ignore) {
            if (item.indexOf(iterator) !== -1) {
                return false;
            }
        }
        return true
    })
}

function readPackageJson(path) {
    try {
        const data = fs.readFileSync(path, { encoding: 'utf8' });
        if (data && typeof data === 'string') {
            return JSON.parse(data)
        }
    } catch (error) {
        console.log('%c 🍦 error: ', 'font-size:20px;background-color: #EA7E5C;color:#fff;', path, error);
    }
}

function getUpdatePkgList(depKeyArr) {
    if(Array.isArray(depKeyArr) && depKeyArr.length <=0) return [];
    let newUpdatePkgList = [];
    let uniDepKeyArr = [...new Set(depKeyArr)];
    if (Array.isArray(cachePkgList)) {
        if (cachePkgList.length <= 0) {
            cachePkgList = uniDepKeyArr;
            newUpdatePkgList = cachePkgList;
        } else {
            newUpdatePkgList = uniDepKeyArr.filter(item => !cachePkgList.includes(item))
            cachePkgList = [...new Set(cachePkgList.concat(uniDepKeyArr))]
        }
    }
    return newUpdatePkgList
}

function updatePkgList(depKeyArr, index) {
    const { decoratorsArr, normalArr } = arrayTypeData(depKeyArr);

    if (Array.isArray(normalArr) && normalArr.length > 0) {
        onesNpmSyncUpdate(normalArr, true)
    }
    if (Array.isArray(decoratorsArr) && decoratorsArr.length > 0) {
        decoratorsArr.forEach(item => {
            onesNpmSyncUpdate(item)
        })
    }
}

const sleep = (time) => new Promise((resolve) => {
    console.log(`🎳🎳🎳 ${chalk.green(`${time / 1000} s`)} 后执行更新操作!`);
    setTimeout(resolve, time);
})

const getExecFileBaseInfo = (abPath) => {
    const { base, dir, ext } = path.parse(abPath);
    const data = [{
        '执行时间': new Date().toISOString(),
        '所在目录': dir,
        '执行文件': base,
        '文件类型': ext,
    }]
    console.table(data);
}

const runScript = async (options) => {
    const pkgGlob = `${options.baseRootPath}/**/**/package.json`;
    let index = 1;
    let execTime = 1000;
    let depKeyArr = [];
    try {
        for await (const path of globby.stream(pkgGlob, { ignore: ['**/node_modules'] })) {
            const packageJson = readPackageJson(path);
            if (packageJson && isPlainObject(packageJson)) {
                const packageDepKey = getPackageJsonDepKey(packageJson, options.ignorePackage);
                if (Array.isArray(packageDepKey) && packageDepKey.length > 0) {
                    depKeyArr = [...depKeyArr, ...packageDepKey]
                }
            }
            const newUpdatePkgList = getUpdatePkgList(depKeyArr);
            if (newUpdatePkgList.length <= 0) {
                continue
            } else {
                getExecFileBaseInfo(path);
                if (index <= 1) {
                    updatePkgList(newUpdatePkgList, index);
                } else {
                    await sleep(execTime * index)
                    updatePkgList(newUpdatePkgList, index);

                }
                index = ++index;
            }
        }
    } catch (error) {
        if (error) {
            if (retryCount < options.maxRetry) {
                console.log('%c 🍞 error: ', 'font-size:20px;background-color: #B03734;color:#fff;', error, '准备重试');
                runScript(options);
                retryCount = ++retryCount;
            }
        }

    }

}

runScript(options);

总结

现在这样就很方便了.随着我本地的项目越来越多.
我只要定期更新一次就可以满足挺久的使用;
而且也不需要全量同步CNPM这么夸张,
只同步使用到的,又能跟进上游!!
有不对之处请留言,谢谢阅读!