Linux内核补丁批量自动下载工具

Posted 2020-08-20 Write the code, change the wor

Linux kernel官网cgit工具不支持按变更代码进行补丁搜索，想到个办法就是把补丁都抓下来，这样可以在本地搜索。花了2个小时写了个小工具，话不多说，直接看效果:

E:\docs\TOOLS\python\patch_spider>python patch_spider.py linux-3.10.y fs/ubifs 2013-08-15:
get patches info...
2016-08-27 eed1a4028c96cabb79747ee01e17b1057b01027c UBIFS: Implement ->migratepage()
2014-11-14 6f1aec53eded9399e6b44cab8c9aa36c65a8f402 UBIFS: fix free log space calculation
2014-11-14 918ecf66a11bb3bdc818a264319dcaf984c11a3f UBIFS: fix a race condition
2014-11-14 c4e70e76860cc84cebd719fbd89637fdd226cf94 UBIFS: remove mst_mutex
2014-07-07 6f02490b96062bdd8a7914e1287a70c5a01d6a3d UBIFS: Remove incorrect assertion in shrink_tnc()
2014-07-07 ac8df9ec7b4e25b87d5a71dfd9af4d8076d66bff UBIFS: fix an mmap and fsync race condition
saving patches into...
[1/6] save fs_ubifs\UBIFS__Implement___gt_migratepage__.patch
[2/6] save fs_ubifs\UBIFS__fix_free_log_space_calculation.patch
[3/6] save fs_ubifs\UBIFS__fix_a_race_condition.patch
[4/6] save fs_ubifs\UBIFS__remove_mst_mutex.patch
[5/6] save fs_ubifs\UBIFS__Remove_incorrect_assertion_in_shrink_tnc__.patch
[6/6] save fs_ubifs\UBIFS__fix_an_mmap_and_fsync_race_condition.patch
done

 

源码

#!/usr/bin/python
# -*- coding: utf-8 -*-
######################################################################
# Purpose:    auto download kernel module patch from offical web
# Useage:    ./patch_spider.py
#            example: patch_spider.py linux-3.10.y fs/ubifs 2013-08-15:2016-11-11
# Version:    Initial    Version    by Alex
######################################################################

import httplib, urllib, urllib2
import sys, os
import datetime
import re

DEBUG = 0
if DEBUG: from pprint import *

class PatchSpider(object):
    def __init__(self):
        pass

    def run(self):
        # main function
        print ‘get patches info...‘
        patch_ids = self.get_patch_id()
        print ‘saving patches into...‘
        ret = self.save_patch_file(patch_ids)    
        print ‘done‘
        
        return 0
        
    def get_patch_id(self):
        # get online data
        response_data = []
        urlstr = "http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/log/%s?h=%s&ofs=%d"
        dt_format = ‘%Y-%m-%d‘
        
        start, end = args[‘date-range‘].split(‘:‘)
        if not start.strip(): start = datetime.datetime.now().strftime(dt_format)
        if not end.strip(): end = datetime.datetime.now().strftime(dt_format)
        dt_start = datetime.datetime.strptime(start, dt_format)
        dt_end = datetime.datetime.strptime(end, dt_format)
        
        for ofs in range(0, 10000, 50):
            url = urlstr % (args[‘module‘], args[‘version‘], ofs)
            req = urllib2.Request(url)

            try: 
                response = urllib2.urlopen(req).read()

                for item in self.extract_from(response):
                    date, id, msg = item
                    dt_date = datetime.datetime.strptime(date, dt_format)
                    if dt_start <= dt_date <= dt_end:
                        print date, id, msg
                        response_data.append(item)
                    else:
                        return response_data
                        
            except urllib2.HTTPError, e:
                print(‘HTTPError = ‘ + str(e.code))
            except urllib2.URLError, e:
                print(‘URLError = ‘ + str(e.reason))
            except httplib.HTTPException, e:
                print(‘HTTPException‘)
            except Exception:
                raise

        return response_data

    def save_patch_file(self, l):
        format = ‘http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/patch/%s?id=%s‘
        folder = self.assemble_path(args[‘module‘])
        
        if os.path.exists(folder):
            print ‘%s existed, please backup your data!‘ %folder
            return -1
        else:
            os.makedirs(folder)
        
        log = ["%s %s %s\n" %(i[0],i[1],i[2]) for i in l]
        open(os.path.join(folder, ‘patches.log‘), ‘w‘).writelines(log)
        
        total = len(l)
        for index, item in enumerate(l):
            
            _,id,msg = item
            patch = format % (args[‘module‘], id)
            file = os.path.join(folder, self.assemble_path(msg) + ‘.patch‘)
                
            urllib.urlretrieve(patch, file)
            print "[%d/%d] save %s" %(index+1, total, file)
    
    
    def extract_from(self, html):
        res = r"""<tr><td>.*?
        <span\ title.*?>(?P<date>\d+-\d+-\d+)</span>.*?
        <a\ href.*?id=(?P<id>\w+)‘>(?P<msg>.*?)</a>.*?
        </td></tr>"""
        rec = re.compile(res, re.DOTALL|re.VERBOSE)
        return re.findall(rec,html)

    def assemble_path(self, msg):
        path = ‘‘
        for c in msg:
            path += c if c.isalnum() else ‘_‘
    
        return path

        
# default args        
args = {
    "version": "linux-3.10.y",
    "module": "fs/ubifs",
    "date-range": "2013-08-15:",
    }
    
if len(sys.argv) > 1:
    args[‘version‘] = sys.argv[1]        
if len(sys.argv) > 2:
    args[‘module‘] = sys.argv[2]
if len(sys.argv) > 3:
    args[‘date-range‘] = sys.argv[3]
    
ret = PatchSpider().run()
sys.exit(ret)