2015-09-25 22:33:55 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"log"
|
|
|
|
"net/url"
|
|
|
|
"strings"
|
|
|
|
"path"
|
|
|
|
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"github.com/cheggaaa/pb"
|
|
|
|
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/image"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Parse new Lets Plays
|
|
|
|
func ParseLetsPlays() {
|
|
|
|
bar := pb.StartNew(0)
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP Page: Request failed (%+v)", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP Page: Document failure (%+v)", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
doc.Find(".postpadding > a").Each(func(i int, s *goquery.Selection) {
|
|
|
|
u, _ := s.Attr("href")
|
|
|
|
ur, _ := url.Parse(u)
|
|
|
|
slug := path.Base(ur.Path)
|
|
|
|
var count int
|
|
|
|
if database.Db.Model(database.LetsPlay{}).Where("slug = ?", slug).Count(&count); count > 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
var LP database.LetsPlay
|
|
|
|
LP.Slug.String = slug
|
|
|
|
LP.Name.String, _ = s.Attr("title")
|
|
|
|
wg.Add(1)
|
|
|
|
go ParseLPPage(LP.Slug.String, &LP, bar)
|
|
|
|
})
|
|
|
|
wg.Wait()
|
|
|
|
bar.FinishPrint("Parsed lets plays")
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) {
|
|
|
|
defer wg.Done()
|
|
|
|
if bar != nil {
|
|
|
|
defer bar.Increment()
|
|
|
|
}
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP %s: Document failure (%+v)", gslug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
pos, _ := doc.Find(".lightbox").Attr("href")
|
|
|
|
LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR LP %s: Error resizing poster: %+v", gslug, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
au, _ := doc.Find(".author > a.avatar").Attr("href")
|
|
|
|
au = strings.TrimSuffix(au, "/")
|
|
|
|
aur, _ := url.Parse(au)
|
|
|
|
aus := path.Base(aur.Path)
|
|
|
|
if aus == "" || aus == "." {
|
|
|
|
log.Printf("ERR LP %s: No author found", gslug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
var AT database.Author
|
|
|
|
database.Db.Where("slug = ?", aus).First(&AT)
|
|
|
|
LP.AuthorID = AT.ID
|
|
|
|
|
|
|
|
if err := database.Db.Create(&LP).Error; err != nil {
|
|
|
|
log.Printf("ERR LP %s: Could not be added to database (%+v)", gslug, err)
|
|
|
|
return
|
|
|
|
} else {
|
|
|
|
log.Printf("SUC LP %s: Added to database", gslug)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update Lets Plays
|
|
|
|
func UpdateLetsPlays() {
|
|
|
|
var LPs []database.LetsPlay
|
|
|
|
var count int
|
|
|
|
database.Db.Find(&LPs).Count(&count)
|
|
|
|
bar := pb.StartNew(count)
|
|
|
|
for _, LP := range LPs {
|
|
|
|
wg.Add(1)
|
|
|
|
go UpdateLP(LP, bar)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
bar.FinishPrint("Updated lets plays")
|
|
|
|
}
|
|
|
|
|
|
|
|
func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) {
|
|
|
|
defer wg.Done()
|
|
|
|
if bar != nil {
|
|
|
|
defer bar.Increment()
|
|
|
|
}
|
|
|
|
if LP.PosterB.Valid == false {
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP %s: Request failed (%+v)", LP.Slug.String, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LP %s: Document failure (%+v)", LP.Slug.String, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
pos, _ := doc.Find(".lightbox").Attr("href")
|
|
|
|
LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR LP %s: Error resizing poster: %+v", LP.Slug.String, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := database.Db.Save(&LP).Error; err != nil {
|
|
|
|
log.Printf("ERR LP %s: Could not be updated in database (%+v)", LP.Slug.String, err)
|
|
|
|
return
|
|
|
|
} else {
|
|
|
|
log.Printf("SUC LP %s: Updated in database", LP.Slug.String)
|
|
|
|
}
|
|
|
|
}
|
2015-09-26 11:45:14 +00:00
|
|
|
}
|