zenermerps/SocialDragon
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Changes instagram parser to use their pagination

Browse source 
Limited to max 5 pages to avoid endless scanning on first start.
If the current page contains at least one media element not in
our database it will check the next page (max. 5 pages).
This commit is contained in:
Andreas Mieke 2017-01-20 03:07:44 +01:00
parent cf135a9af5
commit a97bfc7c14
2 changed files with 26 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
instagram/api.go
View file

@ -69,9 +69,15 @@ type InstagramPostResponse struct {
} `json:"media"` } `json:"media"`
} }
func LoadList() (*InstagramListResponse, error) { func LoadList(token string) (*InstagramListResponse, error) {
var listResponse InstagramListResponse var listResponse InstagramListResponse
res, err := GetHTTPResource("https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1") var url string
if token == "" {
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1"
} else {
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1&max_id=" + token
}
res, err := GetHTTPResource(url)
if err != nil { if err != nil {
return nil, err return nil, err
} }

23
instagram/main.go
View file

@ -8,16 +8,29 @@ import (
func LoadNewInstas() { func LoadNewInstas() {
log.Printf("Loading new Instas...") log.Printf("Loading new Instas...")
list, err := LoadList() token := ""
if err != nil { for i := 0; i < 5; i++ {
log.Printf("Can't load Instagram feed: %+v", err) list, err := LoadList(token)
return if err != nil {
log.Printf("Can't load Instagram feed: %+v", err)
return
}
token = iterList(list)
if token == "" {
break
}
} }
log.Printf("Finished looking for new Instas.")
}
func iterList(list *InstagramListResponse) string {
var token string
for _, image := range list.Tag.Media.Nodes { for _, image := range list.Tag.Media.Nodes {
var count int var count int
if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 { if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 {
continue continue
} }
token = list.Tag.Media.PageInfo.EndCursor
post, err := LoadPost(image.Code) post, err := LoadPost(image.Code)
if err != nil { if err != nil {
log.Printf("Can't load Instagram post %s: %+v", image.Code, err) log.Printf("Can't load Instagram post %s: %+v", image.Code, err)
@ -90,5 +103,5 @@ func LoadNewInstas() {
database.Db.Create(&IT) database.Db.Create(&IT)
} }
} }
log.Printf("Finished looking for new Instas.") return token
} }