golang scrape_shiftjis_page.go

Posted 2021-05-24

package main

import (
	"fmt"
	"io"
	"net/http"

	gq "github.com/PuerkitoBio/goquery"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/transform"
)

var URL = "http://www.kyounoryouri.jp/index.php?flow=recipe_fit_search_result&type=all&keyword=大根&exclude_keyword="

func main() {
	resp, err := http.Get(URL)
	if err != nil {
		panic(err.Error())
	}
	decoder := NewDecoder(resp.Body)
	doc, _ := gq.NewDocumentFromReader(decoder)
	doc.Find(".box_hub_recipe01").Each(func(_ int, s *gq.Selection) {
		fmt.Println(s.Find(".recipeTitle a").Text())
		fmt.Println(s.Find(".recipeTitle a").Attr("href"))
	})
}

func NewDecoder(reader io.Reader) *transform.Reader {
	return transform.NewReader(reader, japanese.ShiftJIS.NewDecoder())
}

json [Golang] golang #golang #snippets中有用的片段

[ fmt ](https://golang.org/pkg/fmt/)
-------
Print any `type`(struct,maps,primitives) with the `key` name 

``` 
fmt.Printf("\n [key] :%+v \n", [value]) 

```

Print Error  

``` 
fmt.Errorf(" \nError: %s", err.Error()) 

```

[ log ](https://golang.org/pkg/log/)
-------------

Print any `type`(struct,maps,primitives) with the `key` name 

``` 
log.Printf("\n [key] :%+v \n", [value]) 

```


[ http  ](https://golang.org/pkg/log/)
-------------

http middleware 

```  
func [middlewareName](h http.Handler) http.Handler {
              return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
              // Do things before request 
           	
              nextMiddleware.ServeHTTP(w, r)
              // Do things after request	  
              })
          } 
```

{
         "http Middleware declaration": {
		"prefix": "md",
		"body": ["func $1(${2:nextMiddleware} http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(${3:w} http.ResponseWriter, ${4:r} *http.Request) {\n\t $5\n\t nextMiddleware.ServeHTTP(${3:w}, ${4:r})\n\t 	  $6\n\t})\n}"],
		"description": "Snippet for http middleware declaration"
	},

	"log.Printf()": {
		"prefix": "lf",
		"body": [
			"log.Printf(\"\\n## ${1:text} ##: %+v\\n\", ${2:value})"
		],
		"description": "log.Printf()"
	},
	"fmt.Printf()": {
		"prefix": "ff",
		"body": [
			"fmt.Printf(\"\\n## ${1:text} ## :%+v \\n\", ${2:value})",
		],
		"description": "log.Printf()"
	}
}