2015-09-25 22:33:55 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"log"
|
|
|
|
"net/url"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
"path"
|
|
|
|
|
|
|
|
"github.com/ChannelMeter/iso8601duration"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"github.com/cheggaaa/pb"
|
|
|
|
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/image"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/youtube"
|
|
|
|
)
|
|
|
|
|
|
|
|
// Parse new Lets Tests
|
|
|
|
func ParseLetsTests() {
|
|
|
|
bar := pb.StartNew(0)
|
|
|
|
ParseLTPage(1, bar)
|
|
|
|
wg.Wait()
|
|
|
|
bar.FinishPrint("Parsed lets tests")
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
|
|
|
defer wg.Done()
|
|
|
|
if bar != nil {
|
|
|
|
defer bar.Increment()
|
|
|
|
}
|
|
|
|
u, _ := s.Find("h1 > a").Attr("href")
|
|
|
|
ur, _ := url.Parse(u)
|
|
|
|
slug := path.Base(ur.Path)
|
|
|
|
var count int
|
|
|
|
if database.Db.Model(database.LetsTest{}).Where("slug = ?", slug).Count(&count); count > 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
var LT database.LetsTest
|
|
|
|
LT.Slug.String = slug
|
|
|
|
LT.Name.String = s.Find("h1 > a").First().Text()
|
|
|
|
res, err := GetHTTPResource(u)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT %s: Request failed (%+v)", slug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT %s: Document failure (%+v)", slug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href")
|
|
|
|
LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR LT %s: Error resizing cover: %+v", slug, err)
|
|
|
|
}
|
|
|
|
au, _ := doc.Find(".author > a.avatar").Attr("href")
|
|
|
|
aur, _ := url.Parse(au)
|
|
|
|
if path.Base(aur.Path) == "" || path.Base(aur.Path) == "." {
|
|
|
|
log.Printf("ERR LT %s: No author found", slug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
var AT database.Author
|
|
|
|
database.Db.Where("slug = ?", path.Base(aur.Path)).First(&AT)
|
|
|
|
LT.AuthorID = AT.ID
|
|
|
|
doc.Find(".article > p").Each(func(i int, s *goquery.Selection) {
|
|
|
|
LT.Descr.String += s.Text() + "\n"
|
|
|
|
})
|
|
|
|
LT.Descr.String = strings.Trim(LT.Descr.String, "\n ")
|
|
|
|
|
|
|
|
yt, _ := doc.Find(".youtube > iframe").Attr("src")
|
|
|
|
ytpath, _ := url.Parse(yt)
|
|
|
|
LT.Youtube.String = path.Base(ytpath.Path)
|
|
|
|
if LT.Youtube.String == "watch" {
|
|
|
|
LT.Youtube.String = ytpath.Query().Get("v")
|
|
|
|
if idx := strings.LastIndex(LT.Youtube.String, "?"); idx != -1 {
|
|
|
|
LT.Youtube.String = LT.Youtube.String[:idx]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ytres, err := youtube.GetVideos([]string {"snippet", "statistics", "status", "contentDetails"}, []string {LT.Youtube.String}, config.C.YoutubeKey)
|
|
|
|
if err != nil || len(ytres.Items) == 0 || ytres.Items[0].Status.UploadStatus != "processed" {
|
|
|
|
log.Printf("WAR LT %s: Video %s is private (%+v)", slug, LT.Youtube.String, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if LT.Descr.String == "" {
|
|
|
|
LT.Descr.String = ytres.Items[0].Snippet.Description
|
|
|
|
}
|
|
|
|
if thumb, ok := ytres.Items[0].Snippet.Thumbnails["maxres"]; ok {
|
|
|
|
LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
|
|
|
|
} else if thumb, ok := ytres.Items[0].Snippet.Thumbnails["high"]; ok {
|
|
|
|
LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR LT %s: Error resizing thumbnail: %+v", slug, err)
|
|
|
|
}
|
|
|
|
LT.Aired, err = time.Parse(time.RFC3339Nano, ytres.Items[0].Snippet.PublishedAt)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT %s: Failed to parse aired", slug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
dur, err := duration.FromString(ytres.Items[0].ContentDetails.Duration)
|
|
|
|
if err == nil {
|
|
|
|
LT.Duration.Int64 = int64(dur.ToDuration().Seconds())
|
|
|
|
} else {
|
|
|
|
log.Printf("ERR LT %s: Failed to parse duration", slug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
LT.Rating.Float64, LT.Votes.Int64 = youtube.GetRatingAndVotesWithRes(ytres)
|
|
|
|
if err := database.Db.Create(<).Error; err != nil {
|
|
|
|
log.Printf("ERR LT %s: Could not be added to databse (%+v)", slug, err)
|
|
|
|
} else {
|
|
|
|
log.Printf("SUC LT %s: Added to database", slug)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseLTPage(page int, bar *pb.ProgressBar) {
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT Page: Request failed (%+v)", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT Page: Document failure (%+v)", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Parse the Episodes
|
|
|
|
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
|
|
|
wg.Add(1)
|
|
|
|
go ParseLT(i, s, bar)
|
|
|
|
})
|
|
|
|
|
|
|
|
_, found := doc.Find(".next").Attr("href")
|
|
|
|
if found {
|
|
|
|
ParseLTPage(page+1, bar)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update Lets Tests
|
|
|
|
func UpdateLetsTests() {
|
|
|
|
var LTs []database.LetsTest
|
|
|
|
var count int
|
|
|
|
database.Db.Find(<s).Count(&count)
|
|
|
|
bar := pb.StartNew(count)
|
|
|
|
for _, LT := range LTs {
|
|
|
|
wg.Add(1)
|
|
|
|
go UpdateLT(LT, bar)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
bar.FinishPrint("Updated lets tests")
|
|
|
|
}
|
|
|
|
|
|
|
|
func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) {
|
|
|
|
defer wg.Done()
|
|
|
|
if bar != nil {
|
|
|
|
defer bar.Increment()
|
|
|
|
}
|
|
|
|
var thumb youtube.Thumb
|
|
|
|
var err error
|
2015-09-29 16:15:07 +00:00
|
|
|
LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)
|
2015-09-29 16:13:34 +00:00
|
|
|
LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR EP %s: Error resizing thumbnail: %+v", LT.Slug.String, err)
|
|
|
|
}
|
|
|
|
if LT.PosterB.Valid == false {
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/" + LT.Slug.String)
|
2015-09-25 22:33:55 +00:00
|
|
|
if err != nil {
|
2015-09-29 16:13:34 +00:00
|
|
|
log.Printf("ERR LT %s: Request failed (%+v)", LT.Slug.String, err)
|
|
|
|
return
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
2015-09-29 16:13:34 +00:00
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR LT %s: Document failure (%+v)", LT.Slug.String, err)
|
|
|
|
return
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
2015-09-29 16:13:34 +00:00
|
|
|
pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href")
|
|
|
|
LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR LT %s: Error resizing cover: %+v", LT.Slug.String, err)
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
2015-09-29 16:13:34 +00:00
|
|
|
}
|
|
|
|
if err := database.Db.Save(<).Error; err != nil {
|
|
|
|
log.Printf("ERR LT %s: Could not be updated in databse (%+v)", LT.Slug.String, err)
|
2015-09-25 22:33:55 +00:00
|
|
|
} else {
|
2015-09-29 16:13:34 +00:00
|
|
|
log.Printf("SUC LT %s: Updated in database", LT.Slug.String)
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
2015-09-26 11:45:14 +00:00
|
|
|
}
|