golang 并发爬虫

Posted 2022-03-27 aloe-n

golang 并发爬虫

之前的一篇文章中展示了一个使用 python 和 aiohttp 搭建的并发爬虫,这篇文章使用 golang 实现同样的功能,旨在理解 python async 异步和 golang 异步编程之间的差别.

代码

package main

import (
    json "encoding/json"
    "fmt"
    ioutil "io/ioutil"
    "net/http"
)

func httpGet(url string) (result []byte, err error) 
    resp, err1 := http.Get(url)
    if err1 != nil 
        err = err1
        return
    

    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    return body, err


func get36kr(page int, ch chan map[string]interface) map[string]interface 
    var mapResult map[string]interface
    url := fmt.Sprintf("https://36kr.com/api/search-column/mainsite?per_page=1&page=%d", page)
    resp, err := httpGet(url)
    if err == nil 
        json.Unmarshal(resp, &mapResult)
    
    return mapResult


func main() 
    // 使用通道进行通信
    ch := make(chan map[string]interface)
    totalPage := 100
    donePageCount := 0
    var itemList []map[string]interface

    for i := 1; i <= totalPage; i++ 
        go func() 
            ch <- get36kr(i, ch)
        ()
    

    // 汇总结果
    for 
        donePageCount++
        data := <-ch
        itemList = append(itemList, data)
        if donePageCount == totalPage 
            str, _ := json.MarshalIndent(&itemList, "", "  ")
            fmt.Println(string(str))
            break