2015-09-25 22:33:55 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/xml"
|
|
|
|
"io/ioutil"
|
|
|
|
"log"
|
|
|
|
"net/url"
|
|
|
|
"path"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/ChannelMeter/iso8601duration"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/image"
|
|
|
|
"git.1750studios.com/gronkhDE/gogronkh/youtube"
|
|
|
|
)
|
|
|
|
|
|
|
|
type RSSItem struct {
|
|
|
|
Link string `xml:"link"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type RSSFeed struct {
|
|
|
|
Items []RSSItem `xml:"channel>item"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseRSSFeeds() {
|
|
|
|
var authors []database.Author
|
|
|
|
database.Db.Find(&authors)
|
|
|
|
for _, AT := range authors {
|
|
|
|
var Feed RSSFeed
|
|
|
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/zeige/" + AT.Slug.String + "/feed")
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(res.Body)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
xerr := xml.Unmarshal(body, &Feed)
|
|
|
|
if xerr != nil {
|
|
|
|
log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, item := range Feed.Items {
|
|
|
|
ur, _ := url.Parse(item.Link)
|
|
|
|
slug := path.Base(ur.Path)
|
|
|
|
var count int
|
|
|
|
if database.Db.Model(database.Episode{}).Where("slug = ?", slug).Count(&count); count > 0 {
|
|
|
|
var EP database.Episode
|
|
|
|
database.Db.Where("slug = ?", slug).First(&EP)
|
|
|
|
wg.Add(1)
|
2016-02-27 11:57:14 +00:00
|
|
|
go UpdateEpisode(EP)
|
2015-09-25 22:33:55 +00:00
|
|
|
} else {
|
|
|
|
wg.Add(1)
|
2016-02-27 11:57:14 +00:00
|
|
|
go ParseFeedEpisode(item.Link)
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
}
|
2015-10-02 13:23:35 +00:00
|
|
|
GenerateSitemap()
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
|
|
|
|
2016-02-27 11:57:14 +00:00
|
|
|
func ParseFeedEpisode(u string) {
|
2015-09-25 22:33:55 +00:00
|
|
|
defer wg.Done()
|
|
|
|
ur, _ := url.Parse(u)
|
|
|
|
slug := path.Base(ur.Path)
|
|
|
|
var EP database.Episode
|
|
|
|
EP.Slug.String = slug
|
|
|
|
var LP database.LetsPlay
|
|
|
|
database.Db.Where("slug = ?", path.Base(path.Dir(ur.Path))).First(&LP)
|
2015-10-15 17:07:54 +00:00
|
|
|
if LP.ID == 0 {
|
|
|
|
wg.Add(1)
|
2015-10-18 19:20:07 +00:00
|
|
|
LP.Slug.String = path.Base(path.Dir(ur.Path))
|
2016-02-27 11:57:14 +00:00
|
|
|
ParseLPPage(LP.Slug.String, &LP)
|
2015-10-18 19:20:07 +00:00
|
|
|
if LP.ID == 0 {
|
|
|
|
return
|
|
|
|
}
|
2015-10-15 17:07:54 +00:00
|
|
|
}
|
2016-02-27 22:10:00 +00:00
|
|
|
if LP.MergeID.Valid == false {
|
|
|
|
EP.LetsPlayID = LP.ID
|
|
|
|
EP.Season.Int64 = 1
|
|
|
|
} else {
|
|
|
|
EP.LetsPlayID = uint(LP.MergeID.Int64)
|
|
|
|
EP.Season.Int64 = LP.MergeSeason.Int64
|
|
|
|
}
|
2015-09-25 22:33:55 +00:00
|
|
|
res, err := GetHTTPResource(u)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR RS %s: Request failed (%+v)", slug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromResponse(res)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR RS %s: Document failure (%+v)", slug, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
EP.Name.String = doc.Find("div.article > h2").First().Text()
|
|
|
|
if EP.Name.String == "" {
|
|
|
|
log.Printf("ERR RS %s: Episode name empty", slug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if match := episodeRegex.FindStringSubmatch(EP.Name.String); len(match) > 0 {
|
|
|
|
num, _ := strconv.Atoi(match[1])
|
|
|
|
EP.Episode.Int64 = int64(num)
|
2016-02-28 13:49:38 +00:00
|
|
|
EP.Name.String = episodeRegex.ReplaceAllString(EP.Name.String, "")
|
|
|
|
EP.Name.String = removeRegex.ReplaceAllString(EP.Name.String, "")
|
2015-09-25 22:33:55 +00:00
|
|
|
} else {
|
2016-02-27 12:14:53 +00:00
|
|
|
log.Printf("WAR RSS %s: Name does not match RegEx", slug)
|
2015-09-25 22:33:55 +00:00
|
|
|
EP.Episode.Int64 = 0
|
2016-02-28 13:49:38 +00:00
|
|
|
EP.Name.String = removeRegex.ReplaceAllString(EP.Name.String, "")
|
2015-09-25 22:33:55 +00:00
|
|
|
}
|
|
|
|
doc.Find(".article > p").Each(func(i int, s *goquery.Selection) {
|
|
|
|
EP.Descr.String += s.Text() + "\n"
|
|
|
|
})
|
|
|
|
EP.Descr.String = strings.Trim(EP.Descr.String, "\n ")
|
2015-11-13 14:08:28 +00:00
|
|
|
EP.AuthorID = LP.AuthorID
|
2015-09-25 22:33:55 +00:00
|
|
|
yt, _ := doc.Find(".youtube > iframe").Attr("src")
|
|
|
|
ytpath, _ := url.Parse(yt)
|
|
|
|
EP.Youtube.String = path.Base(ytpath.Path)
|
|
|
|
if EP.Youtube.String == "watch" {
|
|
|
|
EP.Youtube.String = ytpath.Query().Get("v")
|
|
|
|
if idx := strings.LastIndex(EP.Youtube.String, "?"); idx != -1 {
|
|
|
|
EP.Youtube.String = EP.Youtube.String[:idx]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ytres, err := youtube.GetVideos([]string {"snippet", "statistics", "status", "contentDetails"}, []string {EP.Youtube.String}, config.C.YoutubeKey)
|
|
|
|
if err != nil || len(ytres.Items) == 0 || ytres.Items[0].Status.UploadStatus != "processed" {
|
2016-02-27 12:14:53 +00:00
|
|
|
log.Printf("ERR RS %s: Video %s is private (%s)", slug, EP.Youtube.String, err)
|
2015-09-25 22:33:55 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
if EP.Descr.String == "" {
|
|
|
|
EP.Descr.String = ytres.Items[0].Snippet.Description
|
|
|
|
}
|
|
|
|
if thumb, ok := ytres.Items[0].Snippet.Thumbnails["maxres"]; ok {
|
|
|
|
EP.ThumbS.String, EP.ThumbB.String, err = image.ResizeThumb(thumb.Url)
|
|
|
|
} else if thumb, ok := ytres.Items[0].Snippet.Thumbnails["high"]; ok {
|
|
|
|
EP.ThumbS.String, EP.ThumbB.String, err = image.ResizeThumb(thumb.Url)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("WAR EP %s: Error resizing thumbnail: %+v", slug, err)
|
|
|
|
}
|
|
|
|
EP.Aired, err = time.Parse(time.RFC3339Nano, ytres.Items[0].Snippet.PublishedAt)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("ERR RS %s: Could not parse aired", slug)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
dur, err := duration.FromString(ytres.Items[0].ContentDetails.Duration)
|
|
|
|
if err == nil {
|
|
|
|
EP.Duration.Int64 = int64(dur.ToDuration().Seconds())
|
|
|
|
} else {
|
2016-02-27 12:14:53 +00:00
|
|
|
log.Printf("WAR RSS %s: Could not parse duration", slug)
|
2015-09-25 22:33:55 +00:00
|
|
|
EP.Duration.Int64 = 0
|
|
|
|
}
|
|
|
|
EP.Rating.Float64, EP.Votes.Int64 = youtube.GetRatingAndVotesWithRes(ytres)
|
|
|
|
|
|
|
|
if err := database.Db.Create(&EP).Error; err != nil {
|
|
|
|
log.Printf("ERR RS %s: Could not be added to databse (%+v)", slug, err)
|
|
|
|
}
|
|
|
|
}
|