golang GoLang数据库SQL：从查询中选择未知数量的列。基准测试结果为db_test.go
Posted
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了golang GoLang数据库SQL：从查询中选择未知数量的列。基准测试结果为db_test.go相关的知识，希望对你有一定的参考价值。
package main
import (
"database/sql"
_ "github.com/go-sql-driver/mysql"
"testing"
)
//$ go test -v -bench=. -benchmem .
//testing: warning: no tests to run
//PASS
//BenchmarkMapStringScan 300 4388248 ns/op 416755 B/op 20602 allocs/op
//BenchmarkStrStrScan 300 3990020 ns/op 416629 B/op 20602 allocs/op
//BenchmarkRowMapString 200 7937005 ns/op 1505452 B/op 36304 allocs/op
func BenchmarkMapStringScan(b *testing.B) {
db, err := sql.Open("mysql", "magento-1-8:magento-1-8@tcp(:3306)/magento-1-8")
if err != nil {
b.Fatal(err)
}
defer db.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
rows, err := db.Query(TEST_QUERY)
if err != nil {
b.Fatal(err)
}
defer rows.Close()
columnNames, err := rows.Columns()
if err != nil {
b.Fatal(err)
}
rc := NewMapStringScan(columnNames)
for rows.Next() {
err := rc.Update(rows)
if err != nil {
b.Fatal(err)
}
_ = rc.Get()
}
}
}
func BenchmarkStrStrScan(b *testing.B) {
db, err := sql.Open("mysql", "magento-1-8:magento-1-8@tcp(:3306)/magento-1-8")
if err != nil {
b.Fatal(err)
}
defer db.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
rows, err := db.Query(TEST_QUERY)
if err != nil {
b.Fatal(err)
}
defer rows.Close()
columnNames, err := rows.Columns()
if err != nil {
b.Fatal(err)
}
rc := NewStringStringScan(columnNames)
for rows.Next() {
err := rc.Update(rows)
if err != nil {
b.Fatal(err)
}
_ = rc.Get()
}
}
}
func BenchmarkRowMapString(b *testing.B) {
db, err := sql.Open("mysql", "magento-1-8:magento-1-8@tcp(:3306)/magento-1-8")
if err != nil {
b.Fatal(err)
}
defer db.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
rows, err := db.Query(TEST_QUERY)
if err != nil {
b.Fatal(err)
}
defer rows.Close()
columnNames, err := rows.Columns()
if err != nil {
b.Fatal(err)
}
for rows.Next() {
_, err := rowMapString(columnNames, rows)
if err != nil {
b.Fatal(err)
}
}
}
}
package main
import (
"database/sql"
"fmt"
_ "github.com/go-sql-driver/mysql"
"log"
)
const (
// 1745 rows
// columns are: value_id, entity_type_id, attribute_id, store_id, entity_id, value
TEST_QUERY = `SELECT * FROM catalog_product_entity_varchar`
)
func main() {
db, err := sql.Open("mysql", "magento-1-8:magento-1-8@tcp(:3306)/magento-1-8")
if err != nil {
log.Fatal(err)
}
defer db.Close()
rows, err := db.Query(TEST_QUERY)
fck(err)
defer rows.Close()
columnNames, err := rows.Columns()
fck(err)
rc := NewMapStringScan(columnNames)
for rows.Next() {
// cv, err := rowMapString(columnNames, rows)
// fck(err)
err := rc.Update(rows)
fck(err)
cv := rc.Get()
log.Printf("%#v\n\n", cv)
}
}
/**
using a map
*/
type mapStringScan struct {
// cp are the column pointers
cp []interface{}
// row contains the final result
row map[string]string
colCount int
colNames []string
}
func NewMapStringScan(columnNames []string) *mapStringScan {
lenCN := len(columnNames)
s := &mapStringScan{
cp: make([]interface{}, lenCN),
row: make(map[string]string, lenCN),
colCount: lenCN,
colNames: columnNames,
}
for i := 0; i < lenCN; i++ {
s.cp[i] = new(sql.RawBytes)
}
return s
}
func (s *mapStringScan) Update(rows *sql.Rows) error {
if err := rows.Scan(s.cp...); err != nil {
return err
}
for i := 0; i < s.colCount; i++ {
if rb, ok := s.cp[i].(*sql.RawBytes); ok {
s.row[s.colNames[i]] = string(*rb)
*rb = nil // reset pointer to discard current value to avoid a bug
} else {
return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.colNames[i])
}
}
return nil
}
func (s *mapStringScan) Get() map[string]string {
return s.row
}
/**
using a string slice
*/
type stringStringScan struct {
// cp are the column pointers
cp []interface{}
// row contains the final result
row []string
colCount int
colNames []string
}
func NewStringStringScan(columnNames []string) *stringStringScan {
lenCN := len(columnNames)
s := &stringStringScan{
cp: make([]interface{}, lenCN),
row: make([]string, lenCN*2),
colCount: lenCN,
colNames: columnNames,
}
j := 0
for i := 0; i < lenCN; i++ {
s.cp[i] = new(sql.RawBytes)
s.row[j] = s.colNames[i]
j = j + 2
}
return s
}
func (s *stringStringScan) Update(rows *sql.Rows) error {
if err := rows.Scan(s.cp...); err != nil {
return err
}
j := 0
for i := 0; i < s.colCount; i++ {
if rb, ok := s.cp[i].(*sql.RawBytes); ok {
s.row[j+1] = string(*rb)
*rb = nil // reset pointer to discard current value to avoid a bug
} else {
return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.colNames[i])
}
j = j + 2
}
return nil
}
func (s *stringStringScan) Get() []string {
return s.row
}
// rowMapString was the first implementation but it creates for each row a new
// map and pointers and is considered as slow. see benchmark
func rowMapString(columnNames []string, rows *sql.Rows) (map[string]string, error) {
lenCN := len(columnNames)
ret := make(map[string]string, lenCN)
columnPointers := make([]interface{}, lenCN)
for i := 0; i < lenCN; i++ {
columnPointers[i] = new(sql.RawBytes)
}
if err := rows.Scan(columnPointers...); err != nil {
return nil, err
}
for i := 0; i < lenCN; i++ {
if rb, ok := columnPointers[i].(*sql.RawBytes); ok {
ret[columnNames[i]] = string(*rb)
} else {
return nil, fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, columnNames[i])
}
}
return ret, nil
}
func fck(err error) {
if err != nil {
log.Fatal(err)
}
}
golang查询数据遇到空值问题
golang在查询数据库的时候遇到有空值的列,那么后面的字段提取就会失败，针对这种情况可以有2个办法解决:
1.限制数据库不为空或给个默认值，或是在select查询的时候对空值进行处理，使其查出来的数据不为空
2.采用sql.nullstring进行处理
下面的例子就是sql.sqlstring处理:
package main
import (
_ "github.com/go-sql-driver/mysql"
"fmt"
"database/sql"
)
func nulldel(){
var err error
connect := "dmladmin:dmladmin@tcp(192.168.1.113:3306)/db_admin"
db, err := sql.Open("mysql", connect)
if err != nil{
fmt.Println("connect mysql failed, address = " + connect, err)
} else{
sqlContent := `select
t.statedate,
t.table_schema,
t.table_name,
t1.table_comment,
t.table_rows,
t.lm_table_rows,
case when t.lm_table_rows > 0 then
round((t.table_rows - t.lm_table_rows)*100/t.lm_table_rows,2)
else 0 end growth_rate,
t.gather_interval
from tb_table_grouth_stat t left join information_schema.tables t1
on t.table_schema = t1.table_schema
and t.table_name = t1.table_name
where t.table_schema not in (‘db_admin‘) and statedate = (select max(statedate) from tb_table_grouth_stat)
order by 7 desc limit 10`
rows, err := db.Query(sqlContent)
if err != nil{
fmt.Println(err)
} else{
for rows.Next() {
statedate := sql.NullString{String:"", Valid:false}
table_schema := sql.NullString{String:"", Valid:false}
table_name := sql.NullString{String:"", Valid:false}
table_comment := sql.NullString{String:"", Valid:false}
table_rows := sql.NullString{String:"", Valid:false}
lm_table_rows := sql.NullString{String:"", Valid:false}
growth_rate := sql.NullString{String:"", Valid:false}
gather_interval := sql.NullString{String:"", Valid:false}
rows.Scan(&statedate, &table_schema, &table_name,&table_comment,&table_rows,&lm_table_rows,&growth_rate,&gather_interval)
if ! table_comment.Valid {
table_comment.String="未知"
}
fmt.Println(statedate, table_schema, table_name,table_comment,table_rows,lm_table_rows,growth_rate,gather_interval)
fmt.Println(statedate.String, table_schema.String, table_name.String,table_comment.String,table_rows.String,lm_table_rows.String,growth_rate.String,gather_interval.String)
}
}
}
}
func main() {
nulldel()
}
以上是关于golang GoLang数据库SQL：从查询中选择未知数量的列。基准测试结果为db_test.go的主要内容，如果未能解决你的问题，请参考以下文章
Golang从数据库中查询数据
golang查询数据遇到空值问题
golang配制高性能sql.DB
golang自己定义数据类型查询与插入postgresql中point数据
sql golang查询时，无效的内存地址或nil指针取消引用。
golang 查询数据库操作