Go Web爬虫并发实现

Posted 2021-03-08 大鹏123

题目：Exercise: Web Crawler

直接参考了 https://github.com/golang/tour/blob/master/solutions/webcrawler.go 的实现，不过该代码使用了chan bool来存放子协程是否执行完成，我的代码是使用WaitGroup来让主协程等待子协程执行完成。

完整代码请参考 https://github.com/sxpujs/go-example/blob/master/crawl/web-crawler.go

请注意对于WaitGroup的处理参考了Golang中WaitGroup使用的一点坑

相对原程序增加的代码如下：

var fetched = struct {
    m map[string]error
    sync.Mutex
}{m: map[string]error{}}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
    if _, ok := fetched.m[url]; ok || depth <= 0 {
        return
    }
    body, urls, err := fetcher.Fetch(url)

    fetched.Lock()
    fetched.m[url] = err
    fetched.Unlock()

    if err != nil {
        return
    }
    fmt.Printf("Found: %s %q
", url, body)
    var wg sync.WaitGroup
    for _, u := range urls {
        wg.Add(1)
        go func(url string) {
            defer wg.Done()
            Crawl(url, depth-1, fetcher)
        }(u)
    }
    wg.Wait()
}