优化版本
想看优化版本请移步： 使用Go解析HTTP返回数据为struct并存入数据库的操作
前言
新项目使用Go搭建服务，其中涉及到很多业务数据的构建以及处理的逻辑，笔者也是刚刚开始写Go代码，刚刚开始的时候必然会踩很多坑，这里就记录一下笔者在处理SDK返回的层级数据时遇到的问题以及后续的优化处理。
业务需求描述
从某平台获取到的HTTP原始数据格式如下所示：
{ "request_status": "SUCCESS", "request_id": "xxxxxx", "paging": {}, "adaccounts": [ { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx", "updated_at": "2020-10-28T22:09:24.409Z", "created_at": "2020-08-21T11:00:28.455Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx", "funding_source_ids": [ "xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx", "updated_at": "2020-10-28T21:50:52.923Z", "created_at": "2020-08-21T10:59:07.409Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx", "funding_source_ids": [ "xxx", "fc7cb056-453a-4b3f-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "47ea8129-xxx-xxx-xxx-xxx", "updated_at": "2020-10-28T21:57:05.953Z", "created_at": "2020-08-21T10:57:34.614Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "16412453-xxx-xxx-xxx-xxx", "funding_source_ids": [ "xxx-xxx-4b3f-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx-ed2a-xxx-xxx-xxx", "updated_at": "2020-10-28T21:56:38.374Z", "created_at": "2020-08-21T10:58:35.585Z", "name": "xxx", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx-e008-4353-xxx-xxx", "funding_source_ids": [ "xxx-453a-xxx-8294-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } }, { "sub_request_status": "SUCCESS", "adaccount": { "id": "xxx-fe6c-xxx-a606-xxx", "updated_at": "2020-10-28T21:56:34.531Z", "created_at": "2020-08-21T10:59:53.850Z", "name": "Fashowtime_04_Muyou_EC_SINO_B", "type": "PARTNER", "status": "ACTIVE", "organization_id": "xxx-e008-4353-xxx-xxx", "funding_source_ids": [ "fc7cb056-xxx-4b3f-xxx-xxx" ], "currency": "USD", "timezone": "Asia/Shanghai", } } ] }
现在想要将这些数据分别存入MySQL数据库中。
数据库的表结构如下：
第一版实现思路
思路
一开始，我的思路是：先将这些数据转换为多层嵌套的map结构，然后遍历这个结果，将结果构建成一个切片，这个切片中存放的是只有一层结构的每个账号的数据，最后遍历这个切片，针对切片中每个账号的数据进行处理（还需要将map转换为struct）存入数据库中。
代码
第一版的代码先发出来：
package main import ( "encoding/json" "errors" "fmt" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/mysql" "time" "io/ioutil" "log" "net/http" ) const ( BINano = "2006-01-02 15:04:05.000000000" BIMicro = "2006-01-02 15:04:05.000000" BIMil = "2006-01-02 15:04:05.000" BISec = "2006-01-02 15:04:05" BICST = "2006-01-02 15:04:05 +0800 CST" BIUTC = "2006-01-02 15:04:05 +0000 UTC" BIDate = "2006-01-02" BITime = "15:04:05" ) type AdAccount struct { ID string `json:"id"` Name string `json:"name"` Status string `json:"status"` CreatedAt string `json:"created_at"` CreatedAtTimeStmp int64 `json:"-"` } // 设置表名为 adaccount func (AdAccount) TableName() string { return "adaccount" } // 需要用到的字段 var accountArr = []string{"id", "name", "status", "created_at"} func main() { // 初始化db db, err := gorm.Open("mysql", "root:123@(localhost)/gorm1?charset=utf8mb4&parseTime=True&loc=Local") if err != nil { fmt.Println("err>>> ", err) // panic panic(err) } // defer defer db.Close() // 自动迁移 建表等。。。 db.AutoMigrate(&AdAccount{}) // 刷新token TODO：后续放在Redis中优化一下 token := refreshToken() //fmt.Println("token>>>>>", token) //token := "xxx" // TODO: 账号信息 先用一个固定的 organization 编号写代码 adAccounts := getOrganAdAccounts(baseRqeuest, "xx-e008-xx-xx-xx", token) fmt.Println("adAccounts>>>>>", adAccounts, len(adAccounts)) for _, currM := range adAccounts{ fmt.Printf("%T, %v ",currM,currM) // map[string]interface {}, map[id:xxx name:xxx status:ACTIVE] // currM是一个interface得转一下才能取值！！！ changeM := currM.(map[string]interface{}) fmt.Printf("time1>>>>> %T, %v ",changeM["created_at"],changeM["created_at"]) created_at := changeM["created_at"].(string) // 将时间字符串转换为时间戳 int64 created_at_int,_ := Timestr2Timestamp(created_at) changeM["created_at"] = created_at_int fmt.Printf("time2>>>>> %T, %v ",changeM["created_at"],changeM["created_at"]) fmt.Print("changeM>>> ",changeM) // 然后将map转换为结构体 最终存入数据库中。。。。。。 } } // 判断字符串是否在切片中 func IsContain(strList []string, item string) bool { for _, str := range strList { if str == item { return true } } return false } func getOrganAdAccounts(f func(...string) map[string]interface{}, organizationID string, token string) []interface{} { requestMethod := "GET" url := "https://adsapi.snapchat.com/v1/organizations/" + organizationID + "/adaccounts" ret := f(token, requestMethod, url) //result := NewAdAccountFromJsonString(ret) result := handleHttpRequest(ret, "adaccount") return result } // base func baseRqeuest(args ...string) map[string]interface{} { // token requestMethod url 三个参数是有顺序的！ token := args[0] requestMethod := args[1] url := args[2] client := &http.Client{} // get请求 req, err := http.NewRequest(requestMethod, url, nil) if err != nil { fmt.Println(err) log.Fatal(err) } // 在请求头中加入校验的token req.Header.Set("Authorization", "Bearer "+token) resp, err := client.Do(req) if err != nil { fmt.Println(err) log.Fatal(err) } returnMap, err := ParseResponse(resp) return returnMap } // 解析http请求返回的内容 func ParseResponse(response *http.Response) (map[string]interface{}, error) { var result map[string]interface{} body, err := ioutil.ReadAll(response.Body) // 将 body io数据流转换为 map[string]interface{} 类型返回！ if err == nil { err = json.Unmarshal(body, &result) } return result, err } // refresh token func refreshToken() string { client := &http.Client{} refreshToken := "xxxxxudoucxC2uZInEyQ" clientId := "xxx" clientSecret := "xxx" grantType := "refresh_token" // 字符串拼接 var queryData string queryData = fmt.Sprint("?refresh_token=", refreshToken, "&client_id=", clientId, "&client_secret=", clientSecret, "&grant_type=", grantType) url := "https://accounts.snapchat.com/login/oauth2/access_token" + queryData req, err := http.NewRequest("POST", url, nil) if err != nil { log.Fatal(err) } resp, err := client.Do(req) if err != nil { log.Fatal(err) } // 解析 ret, err := ParseResponse(resp) // 转为string再返回 token := ret["access_token"].(string) return token } // 处理结构化的数据 返回存空接口的切片：里面可以存储任何类型的数据，方便做批量处理 func handleHttpRequest(httpRet map[string]interface{}, handleType string) (result []interface{}) { // 外层的key比里层的key多一个s handleTypes := handleType + "s" for key, val := range httpRet { if key == handleTypes { //fmt.Println(666) mp := val.([]interface{}) // 遍历最外层 for _, orga_val := range mp { fmt.Println(": orga_val>>>>", orga_val) // 判断type取数 switch orga_val.(type) { // 如果是一个字典，继续获取里面的值 case map[string]interface{}: // 转换完后再遍历 orgaValNew := orga_val.(map[string]interface{}) // 遍历 +s 的那个列表 fmt.Println("==========================================================================") for dicKey, dicVal := range orgaValNew { //fmt.Println(dicKey,"<><><>") // 是对应的key才取值! if dicKey == handleType { switch dicVal.(type) { case map[string]interface{}: // 转换完后再遍历 innerDic := dicVal.(map[string]interface{}) // 注意这里必须用临时的map存储要返回的每一个字典！！！ currentMap := make(map[string]interface{}) // 遍历列表中的每一个字典 for innerKey, innerVal := range innerDic { //fmt.Println(innerKey,"-------->",innerVal) // 获取不同类型的结果 // organization直接返回列表 if handleType == "organization" { if innerKey == "id" { id := innerVal.(string) result = append(result, id) } } else if handleType == "adaccount" { // 获取账户信息 flag := IsContain(accountArr, innerKey) if flag == true { //print("adaccount>>>>>>> ") //id := innerVal currentMap[innerKey] = innerVal } } } // 构建结果 result = append(result, currentMap) } } } } } } } return } // 时间字符串转时间戳 func Timestr2Timestamp(str string) (int64, error) { return Timestr2TimestampBasic(str, "", nil) } func Timestr2TimestampBasic(str string, format string, loc *time.Location) (int64, error) { t, err := Timestr2TimeBasic(str, format, loc) if err != nil { return -1., err } return (int64(t.UnixNano()) * 1) / 1e9, nil } func Timestr2TimeBasic(value string, resultFormat string, resultLoc *time.Location) (time.Time, error) { /** - params value: 转换内容字符串 resultFormat: 结果时间格式 resultLoc: 结果时区 */ resultLoc = getLocationDefault(resultLoc) useFormat := []string{ // 可能的转换格式 BINano, BIMicro, BIMil, BISec, BICST, BIUTC, BIDate, BITime, time.RFC3339, time.RFC3339Nano, } var t time.Time for _, usef := range useFormat { tt, error := time.ParseInLocation(usef, value, resultLoc) t = tt if error != nil { continue } break } if t == getTimeDefault(resultLoc) { return t, errors.New("时间字符串格式错误") } if resultFormat == "" { resultFormat = "2006-01-02 15:04:05" } st := t.Format(resultFormat) fixedt, _ := time.ParseInLocation(resultFormat, st, resultLoc) return fixedt, nil } // 获取time默认值, 造一个错误 func getTimeDefault(loc *time.Location) time.Time { loc = getLocationDefault(loc) t, _ := time.ParseInLocation("2006-01-02 15:04:05", "", loc) return t } func getLocationDefault(loc *time.Location) *time.Location { if loc == nil { loc, _ = time.LoadLocation("Local") } return loc }
map转struct的代码
map转struct的代码参考我这篇博客：各种结构相互转换
重要功能说明
请求及解析http的方法
请求及解析http的方法如下（注意我是将解析后的结果转成了 map[string]interface{}），其中还用到了将函数作为参数的传参方式：
func getOrganAdAccounts(f func(...string) map[string]interface{}, organizationID string, token string) []interface{} { requestMethod := "GET" url := "https://adsapi.snapchat.com/v1/organizations/" + organizationID + "/adaccounts" ret := f(token, requestMethod, url) //result := NewAdAccountFromJsonString(ret) result := handleHttpRequest(ret, "adaccount") return result } // base func baseRqeuest(args ...string) map[string]interface{} { // token requestMethod url 三个参数是有顺序的！ token := args[0] requestMethod := args[1] url := args[2] client := &http.Client{} // get请求 req, err := http.NewRequest(requestMethod, url, nil) if err != nil { fmt.Println(err) log.Fatal(err) } // 在请求头中加入校验的token req.Header.Set("Authorization", "Bearer "+token) resp, err := client.Do(req) if err != nil { fmt.Println(err) log.Fatal(err) } returnMap, err := ParseResponse(resp) return returnMap } // 解析http请求返回的内容 func ParseResponse(response *http.Response) (map[string]interface{}, error) { var result map[string]interface{} body, err := ioutil.ReadAll(response.Body) // 将 body io数据流转换为 map[string]interface{} 类型返回！ if err == nil { err = json.Unmarshal(body, &result) } return result, err }
得到的结果如下（转成了多层的map嵌套）：
map[adaccounts:[map[adaccount:map[advertiser_organization_id:16412453-e008-4353-a8da-881ed5170e9c agency_representing_client:false billing_center_id:25440678-19b5-45f1-93bf-8aa6d02513f9 billing_type:IO cell_ids:[f7bc7de9-0edf-482d-a8c4-2468343c1576] client_paying_invoices:false created_at:2020-08-21T11:00:28.455Z currency:USD funding_source_ids:[fc7cb056-453a-4b3f-8294-bcaffc5ee1fb] id:06585df8-81a5-4010-b3b5-8d718d0c4487 lifetime_spend_cap_micro:8e+10 name:Fashowtime_05_Muyou_EC_SINO_B organization_id:16412453-e008-4353-a8da-881ed5170e9c regulations:map[restricted_delivery_signals:false] status:ACTIVE timezone:Asia/Shanghai type:PARTNER updated_at:2020-10-28T22:09:24.409Z] sub_request_status:SUCCESS] map[adaccount:map[advertiser_organization_id:16412453-e008-4353-a8da-881ed5170e9c agency_representing_client:false billing_center_id:25440678-19b5-45f1-93bf-8aa6d02513f9 billing_type:IO cell_ids:[b510b130-b3da-4cba-a42b-c826a2ae346b] client_paying_invoices:false created_at:2020-08-21T10:59:07.409Z currency:USD funding_source_ids:[8b94cfc2-5932-49c8-8865-e4f33a2bad0c fc7cb056-453a-4b3f-8294-bcaffc5ee1fb] id:0f9542e9-56bd-4fae-a6a9-5b9bd48004a6 lifetime_spend_cap_micro:1.5e+11 name:Fashowtime_03_Muyou_EC_SINO_B organization_id:16412453-e008-4353-a8da-881ed5170e9c regulations:map[restricted_delivery_signals:false] status:ACTIVE timezone:Asia/Shanghai type:PARTNER updated_at:2020-10-28T21:50:52.923Z] sub_request_status:SUCCESS] map[adaccount:map[advertiser_organization_id:16412453-e008-4353-a8da-881ed5170e9c agency_representing_client:false billing_center_id:25440678-19b5-45f1-93bf-8aa6d02513f9 billing_type:IO cell_ids:[d231d783-3d33-407a-8e89-36ef492ab25b] client_paying_invoices:false created_at:2020-08-21T10:57:34.614Z currency:USD funding_source_ids:[fc7cb056-453a-4b3f-8294-bcaffc5ee1fb] id:47ea8129-d1e0-4fa3-8df8-e9cab4a64e7b lifetime_spend_cap_micro:1.3e+11 name:Fashowtime_01_Muyou_EC_SINO_B organization_id:16412453-e008-4353-a8da-881ed5170e9c regulations:map[restricted_delivery_signals:false] status:ACTIVE timezone:Asia/Shanghai type:PARTNER updated_at:2020-10-28T21:57:05.953Z] sub_request_status:SUCCESS] map[adaccount:map[advertiser_organization_id:16412453-e008-4353-a8da-881ed5170e9c agency_representing_client:false billing_center_id:25440678-19b5-45f1-93bf-8aa6d02513f9 billing_type:IO cell_ids:[a78bc5b5-b271-41e6-b410-bb089cf3a05c 27a6d3b9-d00e-4923-bb32-aaa5cf6bcd6c] client_paying_invoices:false created_at:2020-08-21T10:58:35.585Z currency:USD funding_source_ids:[fc7cb056-453a-4b3f-8294-bcaffc5ee1fb] id:489ece86-ed2a-4b2f-a697-b470c5f12652 lifetime_spend_cap_micro:2.4e+11 name:Fashowtime_02_Muyou_EC_SINO_B organization_id:16412453-e008-4353-a8da-881ed5170e9c regulations:map[restricted_delivery_signals:false] status:ACTIVE timezone:Asia/Shanghai type:PARTNER updated_at:2020-10-28T21:56:38.374Z] sub_request_status:SUCCESS] map[adaccount:map[advertiser_organization_id:16412453-e008-4353-a8da-881ed5170e9c agency_representing_client:false billing_center_id:25440678-19b5-45f1-93bf-8aa6d02513f9 billing_type:IO cell_ids:[11d7322b-d1de-4ac1-be81-b99306457132 0a9e788b-70ac-4729-8c68-bea42011e3bd] client_paying_invoices:false created_at:2020-08-21T10:59:53.850Z currency:USD funding_source_ids:[fc7cb056-453a-4b3f-8294-bcaffc5ee1fb] id:fa52ec27-fe6c-489d-a606-6a6e19c66690 lifetime_spend_cap_micro:1.7e+11 name:Fashowtime_04_Muyou_EC_SINO_B organization_id:16412453-e008-4353-a8da-881ed5170e9c regulations:map[restricted_delivery_signals:false] status:ACTIVE timezone:Asia/Shanghai type:PARTNER updated_at:2020-10-28T21:56:34.531Z] sub_request_status:SUCCESS]] paging:map[] request_id:5f9a869f00ff099a8b2676f9130001737e616473617069736300016275696c642d31336432356238622d312d3339372d3000010138 request_status:SUCCESS]
处理多层map嵌套的函数
// 处理结构化的数据 返回存空接口的切片：里面可以存储任何类型的数据，方便做批量处理 func handleHttpRequest(httpRet map[string]interface{}, handleType string) (result []interface{}) { // 外层的key比里层的key多一个s handleTypes := handleType + "s" for key, val := range httpRet { if key == handleTypes { //fmt.Println(666) mp := val.([]interface{}) // 遍历最外层 for _, orga_val := range mp { fmt.Println(": orga_val>>>>", orga_val) // 判断type取数 switch orga_val.(type) { // 如果是一个字典，继续获取里面的值 case map[string]interface{}: // 转换完后再遍历 orgaValNew := orga_val.(map[string]interface{}) // 遍历 +s 的那个列表 fmt.Println("==========================================================================") for dicKey, dicVal := range orgaValNew { //fmt.Println(dicKey,"<><><>") // 是对应的key才取值! if dicKey == handleType { switch dicVal.(type) { case map[string]interface{}: // 转换完后再遍历 innerDic := dicVal.(map[string]interface{}) // 注意这里必须用临时的map存储要返回的每一个字典！！！ currentMap := make(map[string]interface{}) // 遍历列表中的每一个字典 for innerKey, innerVal := range innerDic { //fmt.Println(innerKey,"-------->",innerVal) // 获取不同类型的结果 // organization直接返回列表 if handleType == "organization" { if innerKey == "id" { id := innerVal.(string) result = append(result, id) } } else if handleType == "adaccount" { // 获取账户信息 flag := IsContain(accountArr, innerKey) if flag == true { //print("adaccount>>>>>>> ") //id := innerVal currentMap[innerKey] = innerVal } } } // 构建结果 result = append(result, currentMap) } } } } } } } return }
结果如下：
[ map[created_at:2020-08-21T11:00:28.455Z id:06585df8-81a5-4010-b3b5-8d718d0c4487 name:Fashowtime_05_Muyou_EC_SINO_B status:ACTIVE] map[created_at:2020-08-21T10:59:07.409Z id:0f9542e9-56bd-4fae-a6a9-5b9bd48004a6 name:Fashowtime_03_Muyou_EC_SINO_B status:ACTIVE] map[created_at:2020-08-21T10:57:34.614Z id:47ea8129-d1e0-4fa3-8df8-e9cab4a64e7b name:Fashowtime_01_Muyou_EC_SINO_B status:ACTIVE] map[created_at:2020-08-21T10:58:35.585Z id:489ece86-ed2a-4b2f-a697-b470c5f12652 name:Fashowtime_02_Muyou_EC_SINO_B status:ACTIVE] map[created_at:2020-08-21T10:59:53.850Z id:fa52ec27-fe6c-489d-a606-6a6e19c66690 name:Fashowtime_04_Muyou_EC_SINO_B status:ACTIVE] ]
～～～