golang Golang Regexp模式信息

Posted 2021-05-24

// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// DO NOT EDIT. This file is generated by mksyntaxgo from the RE2 distribution.

/*
Package syntax parses regular expressions into parse trees and compiles
parse trees into programs. Most clients of regular expressions will use the
facilities of package regexp (such as Compile and Match) instead of this package.

Syntax

The regular expression syntax understood by this package when parsing with the Perl flag is as follows.
Parts of the syntax can be disabled by passing alternate flags to Parse.


Single characters:
  .              any character, possibly including newline (flag s=true)
  [xyz]          character class
  [^xyz]         negated character class
  \d             Perl character class
  \D             negated Perl character class
  [[:alpha:]]    ASCII character class
  [[:^alpha:]]   negated ASCII character class
  \pN            Unicode character class (one-letter name)
  \p{Greek}      Unicode character class
  \PN            negated Unicode character class (one-letter name)
  \P{Greek}      negated Unicode character class

Composites:
  xy             x followed by y
  x|y            x or y (prefer x)

Repetitions:
  x*             zero or more x, prefer more
  x+             one or more x, prefer more
  x?             zero or one x, prefer one
  x{n,m}         n or n+1 or ... or m x, prefer more
  x{n,}          n or more x, prefer more
  x{n}           exactly n x
  x*?            zero or more x, prefer fewer
  x+?            one or more x, prefer fewer
  x??            zero or one x, prefer zero
  x{n,m}?        n or n+1 or ... or m x, prefer fewer
  x{n,}?         n or more x, prefer fewer
  x{n}?          exactly n x

Implementation restriction: The counting forms x{n,m}, x{n,}, and x{n}
reject forms that create a minimum or maximum repetition count above 1000.
Unlimited repetitions are not subject to this restriction.

Grouping:
  (re)           numbered capturing group (submatch)
  (?P<name>re)   named & numbered capturing group (submatch)
  (?:re)         non-capturing group
  (?flags)       set flags within current group; non-capturing
  (?flags:re)    set flags during re; non-capturing

  Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z). The flags are:

  i              case-insensitive (default false)
  m              multi-line mode: ^ and $ match begin/end line in addition to begin/end text (default false)
  s              let . match \n (default false)
  U              ungreedy: swap meaning of x* and x*?, x+ and x+?, etc (default false)

Empty strings:
  ^              at beginning of text or line (flag m=true)
  $              at end of text (like \z not Perl's \Z) or line (flag m=true)
  \A             at beginning of text
  \b             at ASCII word boundary (\w on one side and \W, \A, or \z on the other)
  \B             not at ASCII word boundary
  \z             at end of text

Escape sequences:
  \a             bell (== \007)
  \f             form feed (== \014)
  \t             horizontal tab (== \011)
  \n             newline (== \012)
  \r             carriage return (== \015)
  \v             vertical tab character (== \013)
  \*             literal *, for any punctuation character *
  \123           octal character code (up to three digits)
  \x7F           hex character code (exactly two digits)
  \x{10FFFF}     hex character code
  \Q...\E        literal text ... even if ... has punctuation

Character class elements:
  x              single character
  A-Z            character range (inclusive)
  \d             Perl character class
  [:foo:]        ASCII character class foo
  \p{Foo}        Unicode character class Foo
  \pF            Unicode character class F (one-letter name)

Named character classes as character class elements:
  [\d]           digits (== \d)
  [^\d]          not digits (== \D)
  [\D]           not digits (== \D)
  [^\D]          not not digits (== \d)
  [[:name:]]     named ASCII class inside character class (== [:name:])
  [^[:name:]]    named ASCII class inside negated character class (== [:^name:])
  [\p{Name}]     named Unicode property inside character class (== \p{Name})
  [^\p{Name}]    named Unicode property inside negated character class (== \P{Name})

Perl character classes (all ASCII-only):
  \d             digits (== [0-9])
  \D             not digits (== [^0-9])
  \s             whitespace (== [\t\n\f\r ])
  \S             not whitespace (== [^\t\n\f\r ])
  \w             word characters (== [0-9A-Za-z_])
  \W             not word characters (== [^0-9A-Za-z_])

ASCII character classes:
  [[:alnum:]]    alphanumeric (== [0-9A-Za-z])
  [[:alpha:]]    alphabetic (== [A-Za-z])
  [[:ascii:]]    ASCII (== [\x00-\x7F])
  [[:blank:]]    blank (== [\t ])
  [[:cntrl:]]    control (== [\x00-\x1F\x7F])
  [[:digit:]]    digits (== [0-9])
  [[:graph:]]    graphical (== [!-~] == [A-Za-z0-9!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~])
  [[:lower:]]    lower case (== [a-z])
  [[:print:]]    printable (== [ -~] == [ [:graph:]])
  [[:punct:]]    punctuation (== [!-/:-@[-`{-~])
  [[:space:]]    whitespace (== [\t\n\v\f\r ])
  [[:upper:]]    upper case (== [A-Z])
  [[:word:]]     word characters (== [0-9A-Za-z_])
  [[:xdigit:]]   hex digit (== [0-9A-Fa-f])

*/
package syntax

纯golang爬虫实战(二）

接上一篇文章https://www.cnblogs.com/pu369/p/12202845.html只讲了原理，抽时间写个了实用版，将员工信息爬取到一个TXT文档中，以便于查询，上代码：

//纯golang爬虫
package main

import (
    "bytes"
    "fmt"
    "io/ioutil"
    "net/http"
    "net/http/cookiejar"
    "regexp"
    "strings"
)

type MySpider struct {
    indexUrl string
    cleint   *http.Client
    buf      *bytes.Buffer
}

//登录，用GET代替POST请求
func (this MySpider) login() (string, error) {
    //访问首页
    resp, err := this.cleint.Get(this.indexUrl)
    //访问登录页
    resp, err = this.cleint.Get("http://192.168.13.1:8080/login/auth?name=XX&password=XX&scurity=s&type=0&typeField=0")
    body, err := ioutil.ReadAll(resp.Body)
    defer resp.Body.Close()
    if err != nil {
        return "err", err
    }
    //fmt.Print(string(body))
    //trimbody := []byte(trimHtml(string(body)))
    //循环访问
    this.saveall()
    return string(body), err
}

//循环下载所有内容、保存到bytes.buffer,最后统一写入文件（如果数据量太大可能会崩溃）
func (this MySpider) saveall() (string, error) {
    //最小id是2，但id是8位字符串，不足8位在前面补0
    for id := 2; id < 20000; id++ {
        idstr := fmt.Sprintf("%08d", id)
        //fmt.Println(idstr)

        //员工记录主页面
        url := "http://192.168.13.1:8080/browse/basicinfo_p.jsp?rtpage=psnfrm&pid=" + idstr + "&func=0297&userbase=Usr"
        this.saveone(url, idstr)
    }
    //fmt.Print("buf:", this.buf.String())
    //保存到文件
    err := ioutil.WriteFile("hrp.txt", this.buf.Bytes(), 0644)
    if err != nil {
        return "err", err
    }
    return "", err
}

//下载某人员的主页面内容、保存到bytes.buffer
func (this MySpider) saveone(url, idstr string) (string, error) {
    resp, err := this.cleint.Get(url)
    if err != nil {
        return "err", err
    }
    body, err := ioutil.ReadAll(resp.Body)
    defer resp.Body.Close()
    stringbody := string(body)
    //fmt.Print(string(body))
    //判断主页面是否包含字样：Apache Tomcat/5.0.19 - Error report |  HTTP Status 500
    if ko := strings.Contains(stringbody, "Apache"); !ko {
        //主页面正常，则保存
        this.buf.Write([]byte(idstr + "
"))
        trimbody := []byte(trimHtml(stringbody))
        this.buf.Write(trimbody)
        this.buf.Write([]byte("
"))
        //有主页面，则下载辅助页面
        //员工记录附加页面－1学历
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A04&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－2岗位
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A17&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－3简历
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A19&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－4合同
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=AZ3&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－5流动
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A16&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－6关系
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A79&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－7家庭
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A82&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－8聘任
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=AZT&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－9职务
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A07&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－10专业
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A10&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－11工人
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A13&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－12奖励
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A28&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        //员工记录附加页面－13惩罚
        url = "http://192.168.13.1:8080/browse/subsetinfo_p.jsp?ref=A29&pid=" + idstr + "&userbase=Usr"
        this.saveonemore(url)
        this.buf.Write([]byte("

"))
    }
    return "", err
}

//下载某人员的辅助页面内容、保存到bytes.buffer
func (this MySpider) saveonemore(url string) (string, error) {
    resp, err := this.cleint.Get(url)
    if err != nil {
        return "err", err
    }
    body, err := ioutil.ReadAll(resp.Body)
    defer resp.Body.Close()
    stringbody := string(body)
    trimbody := []byte(trimHtml(stringbody))
    this.buf.Write(trimbody)
    this.buf.Write([]byte("
"))
    return "", err
}

//去除Html标签
func trimHtml(src string) string {
    //将HTML标签全转换成小写
    re, _ := regexp.Compile("\\<[\\S\\s]+?\\>")
    src = re.ReplaceAllStringFunc(src, strings.ToLower)
    //去除STYLE
    re, _ = regexp.Compile("\\<style[\\S\\s]+?\\</style\\>")
    src = re.ReplaceAllString(src, " ")
    //去除SCRIPT
    re, _ = regexp.Compile("\\<script[\\S\\s]+?\\</script\\>")
    src = re.ReplaceAllString(src, " ")
    //去除所有尖括号内的HTML代码，并换成换行符
    re, _ = regexp.Compile("\\<[\\S\\s]+?\\>")
    src = re.ReplaceAllString(src, " ")
    //去除&nbsp
    re, _ = regexp.Compile("&nbsp")
    src = re.ReplaceAllString(src, " ")
    //去除连续的换行符
    re, _ = regexp.Compile("\\s{2,}")
    src = re.ReplaceAllString(src, " |  ")
    return strings.TrimSpace(src)
}

//运行
func (this MySpider) run() string {
    //生成可复用的client
    var client http.Client
    jar, err := cookiejar.New(nil)
    if err != nil {
        panic(err)
    }
    client.Jar = jar
    this.cleint = &client
    //登录，用GET代替POST请求
    this.login()
    return ""
}

func main() {
    //爬虫实例
    ms := new(MySpider)
    //入口地址http://192.168.13.1:8080
    ms.indexUrl = "http://192.168.13.1:8080"
    ms.buf = bytes.NewBuffer([]byte{})
    ms.run()
}