golang Amazon S3使用AWS SDK for Go选择CSV示例

Posted 2021-05-24

package main

import (
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

func main() {
	sess := session.Must(session.NewSession())
	svc := s3.New(
		sess,
		aws.NewConfig().WithRegion("ap-northeast-1"),
	)


	params := &s3.SelectObjectContentInput{
		Bucket:          aws.String("s3-select-sampleyy"),
		Key:             aws.String("FIFA2018_Statistics.csv"),
		ExpressionType:  aws.String(s3.ExpressionTypeSql),
		Expression:      aws.String("SELECT * FROM S3Object LIMIT 10"),
		RequestProgress: &s3.RequestProgress{},
		InputSerialization: &s3.InputSerialization{
			CompressionType: aws.String("NONE"),
			CSV: &s3.CSVInput{
				FileHeaderInfo: aws.String(s3.FileHeaderInfoUse),
			},
		},
		OutputSerialization: &s3.OutputSerialization{
			CSV: &s3.CSVOutput{},
		},
	}

	resp, err := svc.SelectObjectContent(params)
	if err != nil {
		exitErrorf("failed making API request", err)
	}
	defer resp.EventStream.Close()

	for event := range resp.EventStream.Events() {
	 // v, ok := event.(*s3.RecordsEvent)
		// if ok {
		// 	fmt.Println(string(v.Payload))
		// }
		switch v := event.(type) {
		case *s3.RecordsEvent:
		  // s3.RecordsEvent.Records is a byte slice of select records
			fmt.Println(string(v.Payload))
		case *s3.StatsEvent:
		  // s3.StatsEvent contains information on the data that’s processed
			fmt.Println("Processed", *v.Details.BytesProcessed, "bytes")
			fmt.Println("Returned", *v.Details.BytesReturned, "bytes")
			fmt.Println("Scanned", *v.Details.BytesScanned, "bytes")
		case *s3.EndEvent:
			fmt.Println("SelectObjectContent completed")
		}
	}

	if err := resp.EventStream.Err(); err != nil {
		exitErrorf("failed to read from SelectObject EventStream, %v", err)
	}
}

golang Amazon S3使用AWS SDK for Go选择带有JSON的示例

package main

import (
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

func main() {
	sess := session.Must(session.NewSession())
	svc := s3.New(
		sess,
		aws.NewConfig().WithRegion("ap-northeast-1"),
	)

	params := &s3.SelectObjectContentInput{
		Bucket:         aws.String("s3-select-sampleyy"),
		Key:            aws.String("sample.jsonl"),
		ExpressionType: aws.String(s3.ExpressionTypeSql),
		Expression:     aws.String("SELECT * FROM S3Object LIMIT 10"),
		InputSerialization: &s3.InputSerialization{
			CompressionType: aws.String("NONE"),
			JSON: &s3.JSONInput{
				Type: aws.String("Lines"),
			},
		},
		OutputSerialization: &s3.OutputSerialization{
			JSON: &s3.JSONOutput{},
		},
		//// You can also output with csv.
		//OutputSerialization: &s3.OutputSerialization{
		//	CSV: &s3.CSVOutput{
		//		//RecordDelimiter: aws.String("\n"),
		//		//FieldDelimiter:  aws.String(","),
		//	},
		//},
	}

	resp, err := svc.SelectObjectContent(params)
	if err != nil {
		exitErrorf("failed making API request", err)
	}
	defer resp.EventStream.Close()

	for event := range resp.EventStream.Events() {
		switch v := event.(type) {
		case *s3.RecordsEvent:
			fmt.Println(string(v.Payload))
		case *s3.StatsEvent:
			fmt.Println("Processed", *v.Details.BytesProcessed, "bytes")
			fmt.Println("Returned", *v.Details.BytesReturned, "bytes")
			fmt.Println("Scanned", *v.Details.BytesScanned, "bytes")
		case *s3.EndEvent:
			fmt.Println("SelectObjectContent completed")
		}
	}

	if err := resp.EventStream.Err(); err != nil {
		exitErrorf("failed to read from SelectObject EventStream, %v", err)
	}
}