From 0a69d4a9f2299868df0b8599e0b55d1a63c19e24 Mon Sep 17 00:00:00 2001 From: Andreas Mieke Date: Tue, 17 Jan 2017 23:38:35 +0100 Subject: [PATCH] Initial commit Implemented getters for: * Snapchat * Twitter * Instagram Also implemented config and database modules. --- .gitignore | 61 +++++++++++++++ config/config.go | 64 ++++++++++++++++ database/main.go | 67 +++++++++++++++++ instagram/api.go | 149 +++++++++++++++++++++++++++++++++++++ instagram/http.go | 33 +++++++++ instagram/main.go | 94 ++++++++++++++++++++++++ snapchat/api.go | 163 +++++++++++++++++++++++++++++++++++++++++ snapchat/http.go | 48 ++++++++++++ snapchat/main.go | 57 ++++++++++++++ socialdragon/main.go | 33 +++++++++ socialdragon/test.toml | 33 +++++++++ twitter/downloader.go | 58 +++++++++++++++ twitter/http.go | 33 +++++++++ twitter/main.go | 120 ++++++++++++++++++++++++++++++ 14 files changed, 1013 insertions(+) create mode 100644 .gitignore create mode 100644 config/config.go create mode 100644 database/main.go create mode 100644 instagram/api.go create mode 100644 instagram/http.go create mode 100644 instagram/main.go create mode 100644 snapchat/api.go create mode 100644 snapchat/http.go create mode 100644 snapchat/main.go create mode 100644 socialdragon/main.go create mode 100644 socialdragon/test.toml create mode 100644 twitter/downloader.go create mode 100644 twitter/http.go create mode 100644 twitter/main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f3fafc --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ + +# Created by https://www.gitignore.io/api/go,osx + +### Go ### +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# External packages folder +vendor/ + + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon +# Thumbnails +._* +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# End of https://www.gitignore.io/api/go,osx diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..a09c56b --- /dev/null +++ b/config/config.go @@ -0,0 +1,64 @@ +package config + +import ( + "bytes" + "io/ioutil" + + "github.com/BurntSushi/toml" +) + +type Config struct { + DatabaseConnection string + BindAddress string + AssetsDirectory string + TemplatesDirectory string + ContentDirectory string + ContentWebDirectory string + Snapchat Snapchat + Twitter Twitter + Instagram Instagram +} + +type Instagram struct { + Tag string +} + +type Twitter struct { + ConsumerKey string + ConsumerSecret string + OAuthToken string + OAuthTokenSecret string + Filter []string +} + +type Snapchat struct { + ApiBase string + UserAgent string + UserName string + GetConversations SnapchatEndpoint + GetBlob SnapchatEndpoint + MarkAsSeen SnapchatEndpoint +} + +type SnapchatEndpoint struct { + Uuid string + ClientAuthToken string + RequestToken string + Timestamp string +} + +var C Config + +func LoadConfig(path string) error { + _, e := toml.DecodeFile(path, &C) + return e +} +func WriteConfig(path string) error { + buf := new(bytes.Buffer) + err := toml.NewEncoder(buf).Encode(C) + if err != nil { + return err + } + err = ioutil.WriteFile(path, buf.Bytes(), 0644) + return err +} diff --git a/database/main.go b/database/main.go new file mode 100644 index 0000000..c4cfe25 --- /dev/null +++ b/database/main.go @@ -0,0 +1,67 @@ +package database + +import ( + "log" + "time" + + "git.1750studios.com/AniNite/SocialDragon/config" + + "github.com/jinzhu/gorm" + _ "github.com/lib/pq" +) + +type Service uint + +const ( + Snapchat Service = iota + Twitter + Instagram +) + +type State uint + +const ( + Inbox State = iota + Approved + Rejected +) + +type Item struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time + + UserID uint `sql:"index"` + Service Service + State State + IsVideo bool + Path string + OriginalID string +} + +type User struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time + + Name string + DisplayName string + Service Service + Blocked bool +} + +var Db *gorm.DB + +func InitDb() { + var err error + Db, err = gorm.Open("postgres", config.C.DatabaseConnection) + if err != nil { + log.Fatalf("Database error: %+v", err) + } + Db.LogMode(false) + Db.SingularTable(true) + Db.Model(&Item{}).AddForeignKey("user_id", "user(id)", "RESTRICT", "RESTRICT") + Db.AutoMigrate(&Item{}, &User{}) +} diff --git a/instagram/api.go b/instagram/api.go new file mode 100644 index 0000000..867cb17 --- /dev/null +++ b/instagram/api.go @@ -0,0 +1,149 @@ +package instagram + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "io/ioutil" + "log" + "os" + + "github.com/disintegration/imaging" + + "git.1750studios.com/AniNite/SocialDragon/config" +) + +type InstagramListResponse struct { + Tag struct { + Media struct { + Nodes []struct { + ID string `json:"id"` + Caption string `json:"caption"` + DisplaySrc string `json:"display_src"` + IsVideo bool `json:"is_video"` + Owner struct { + ID string `json:"id"` + } `json:"owner"` + ThumbnailSrc string `json:"thumbnail_src"` + Code string `json:"code"` + Date int `json:"date"` + } `json:"nodes"` + PageInfo struct { + EndCursor string `json:"end_cursor"` + HasNextPage bool `json:"has_next_page"` + StartCursor string `json:"start_cursor"` + HasPreviousPage bool `json:"has_previous_page"` + } `json:"page_info"` + Count int `json:"count"` + } `json:"media"` + } `json:"tag"` +} + +type InstagramPostResponse struct { + Media struct { + Owner struct { + ProfilePicURL string `json:"profile_pic_url"` + HasBlockedViewer bool `json:"has_blocked_viewer"` + Username string `json:"username"` + FullName string `json:"full_name"` + RequestedByViewer bool `json:"requested_by_viewer"` + FollowedByViewer bool `json:"followed_by_viewer"` + IsPrivate bool `json:"is_private"` + ID string `json:"id"` + BlockedByViewer bool `json:"blocked_by_viewer"` + IsUnpublished bool `json:"is_unpublished"` + } `json:"owner"` + CaptionIsEdited bool `json:"caption_is_edited"` + VideoURL string `json:"video_url"` + DisplaySrc string `json:"display_src"` + CommentsDisabled bool `json:"comments_disabled"` + Code string `json:"code"` + IsAd bool `json:"is_ad"` + IsVideo bool `json:"is_video"` + Caption string `json:"caption"` + ID string `json:"id"` + Date int `json:"date"` + VideoViews int `json:"video_views"` + } `json:"media"` +} + +func LoadList() (*InstagramListResponse, error) { + var listResponse InstagramListResponse + res, err := GetHTTPResource("https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1") + if err != nil { + return nil, err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + jerr := json.Unmarshal(body, &listResponse) + if jerr != nil { + return nil, jerr + } + return &listResponse, nil +} + +func LoadPost(code string) (*InstagramPostResponse, error) { + var postResponse InstagramPostResponse + res, err := GetHTTPResource("https://www.instagram.com/p/" + code + "/?__a=1") + if err != nil { + return nil, err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + jerr := json.Unmarshal(body, &postResponse) + if jerr != nil { + return nil, jerr + } + return &postResponse, nil +} + +func ImageNameGenerator(seed string) (string, string) { + seedBytes := []byte(seed) + sha256Bytes := sha256.Sum256(seedBytes) + hash := hex.EncodeToString(sha256Bytes[:]) + folders := config.C.ContentDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + urls := config.C.ContentWebDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + if err := os.MkdirAll(folders, 0775); err != nil { + log.Fatalf("FAT Could not create ContentDirectory, error: %+v", err) + } + finalPath := folders + hash + finalUrl := urls + hash + return finalPath, finalUrl +} + +func DownloadMedia(data io.Reader, path string, video bool) (string, error) { + if video { + ext := ".mp4" + blob, err := ioutil.ReadAll(data) + if err != nil { + return "", err + } + if len(blob) == 0 { + return "", errors.New("Empty response") + } + err = ioutil.WriteFile(path+ext, blob, 0644) + if err != nil { + return "", err + } + return ext, nil + } else { + ext := ".jpg" + image, err := imaging.Decode(data) + if err != nil { + return "", err + } + err = imaging.Save(image, path+ext) + if err != nil { + return "", err + } + return ext, nil + } +} diff --git a/instagram/http.go b/instagram/http.go new file mode 100644 index 0000000..148d437 --- /dev/null +++ b/instagram/http.go @@ -0,0 +1,33 @@ +package instagram + +import ( + "errors" + "log" + "net/http" + "strconv" +) + +var sem = make(chan byte, 2) +var client = &http.Client{} + +func GetHTTPResource(u string) (*http.Response, error) { + sem <- 1 + req, err := http.NewRequest("GET", u, nil) + if err != nil { + log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) + <-sem + return nil, err + } + res, err := client.Do(req) + if err != nil { + <-sem + return nil, err + } + if res.StatusCode != 200 { + res.Body.Close() + <-sem + return nil, errors.New(strconv.Itoa(res.StatusCode)) + } + <-sem + return res, nil +} diff --git a/instagram/main.go b/instagram/main.go new file mode 100644 index 0000000..01e472f --- /dev/null +++ b/instagram/main.go @@ -0,0 +1,94 @@ +package instagram + +import ( + "log" + + "git.1750studios.com/AniNite/SocialDragon/database" +) + +func LoadNewInstas() { + log.Printf("Loading new Instas...") + list, err := LoadList() + if err != nil { + log.Printf("Can't load Instagram feed: %+v", err) + return + } + for _, image := range list.Tag.Media.Nodes { + var count int + if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 { + continue + } + post, err := LoadPost(image.Code) + if err != nil { + log.Printf("Can't load Instagram post %s: %+v", image.Code, err) + continue + } + if post.Media.IsAd { + continue + } + var US database.User + if database.Db.Model(database.User{}).Where("name = ? AND service = ?", post.Media.Owner.Username, database.Instagram).First(&US).Count(&count); count == 0 { + US.DisplayName = post.Media.Owner.FullName + US.Name = post.Media.Owner.Username + US.Service = database.Instagram + US.Blocked = false + database.Db.Create(&US) + } + if post.Media.IsVideo { + log.Printf("Found video %s from %s", post.Media.Code, post.Media.Owner.Username) + name, uname := ImageNameGenerator(post.Media.Code) + res, err := GetHTTPResource(post.Media.VideoURL) + if err != nil { + log.Printf("Can't load video %s: %+v", post.Media.Code, err) + continue + } + defer res.Body.Close() + ext, err := DownloadMedia(res.Body, name, true) + if err != nil { + log.Printf("Can't load video %s: %+v", post.Media.Code, err) + continue + } + log.Printf("Loaded video %s, location %s!", post.Media.Code, uname+ext) + var IT database.Item + IT.UserID = US.ID + IT.Service = database.Instagram + if US.Blocked { + IT.State = database.Rejected + } else { + IT.State = database.Inbox + } + IT.IsVideo = true + IT.Path = uname + ext + IT.OriginalID = post.Media.Code + database.Db.Create(&IT) + } else { + log.Printf("Found picture %s from %s", post.Media.Code, post.Media.Owner.Username) + name, uname := ImageNameGenerator(post.Media.Code) + res, err := GetHTTPResource(post.Media.DisplaySrc) + if err != nil { + log.Printf("Can't load picture %s: %+v", post.Media.Code, err) + continue + } + defer res.Body.Close() + ext, err := DownloadMedia(res.Body, name, false) + if err != nil { + log.Printf("Can't load picture %s: %+v", post.Media.Code, err) + continue + } + log.Printf("Loaded picture %s, location %s!", post.Media.Code, uname+ext) + var IT database.Item + IT.UserID = US.ID + IT.Service = database.Instagram + if US.Blocked { + IT.State = database.Rejected + } else { + IT.State = database.Inbox + } + IT.IsVideo = false + IT.Path = uname + ext + IT.OriginalID = post.Media.Code + database.Db.Create(&IT) + } + } + log.Printf("Finished looking for new Instas.") +} diff --git a/snapchat/api.go b/snapchat/api.go new file mode 100644 index 0000000..527d3b6 --- /dev/null +++ b/snapchat/api.go @@ -0,0 +1,163 @@ +package snapchat + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "io/ioutil" + "log" + "os" + + "github.com/disintegration/imaging" + + "git.1750studios.com/AniNite/SocialDragon/config" +) + +type Conversations struct { + ConversationsResponse []ConversationsResponse `json:"conversations_response"` +} + +type ConversationsResponse struct { + PendingReceivedSnaps []Snap `json:"pending_received_snaps"` +} + +type Snap struct { + Id string `json:"id"` + Username string `json:"sn"` + Movie int `json:"m"` + Rotation int `json:"mo"` +} + +func GetConversations() (*Conversations, error) { + var conversations Conversations + headers := map[string]string{ + "X-Snapchat-UUID": config.C.Snapchat.GetConversations.Uuid, + "X-Snapchat-Client-Auth-Token": config.C.Snapchat.GetConversations.ClientAuthToken, + } + data := map[string]string{ + "checksum_dict": "{}", + "friends_request": "{\"friends_sync_token\":\"3\"}", + "req_token": config.C.Snapchat.GetConversations.RequestToken, + "timestamp": config.C.Snapchat.GetConversations.Timestamp, + "username": config.C.Snapchat.UserName, + } + res, err := GetHTTPResource("/loq/conversations", headers, data) + if err != nil { + return nil, err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + jerr := json.Unmarshal(body, &conversations) + if jerr != nil { + return nil, jerr + } + return &conversations, nil +} + +func GetBlob(snap Snap) (string, error) { + headers := map[string]string{ + "X-Snapchat-UUID": config.C.Snapchat.GetBlob.Uuid, + "X-Snapchat-Client-Auth-Token": config.C.Snapchat.GetBlob.ClientAuthToken, + } + data := map[string]string{ + "id": snap.Id, + "req_token": config.C.Snapchat.GetBlob.RequestToken, + "timestamp": config.C.Snapchat.GetBlob.Timestamp, + "username": config.C.Snapchat.UserName, + } + res, err := GetHTTPResource("/bq/blob", headers, data) + if err != nil { + return "", err + } + name, uname := ImageNameGenerator(snap.Id) + defer res.Body.Close() + ext, err := RotateImage(snap, res.Body, name) + if err != nil { + return "", err + } + return uname + ext, nil +} + +func MarkAsSeen(snap Snap, screenshotted bool) error { + headers := map[string]string{ + "X-Snapchat-UUID": config.C.Snapchat.MarkAsSeen.Uuid, + "X-Snapchat-Client-Auth-Token": config.C.Snapchat.MarkAsSeen.ClientAuthToken, + } + data := map[string]string{ + "req_token": config.C.Snapchat.MarkAsSeen.RequestToken, + "timestamp": config.C.Snapchat.MarkAsSeen.Timestamp, + "username": config.C.Snapchat.UserName, + } + if screenshotted { + data["json"] = "{\"" + snap.Id + "\":{\"t\":1467469712.44128,\"replayed\":0,\"c\":1,\"stack_id\":\"E437851F-2AA8-4C6C-AE11-5FB49FBF93C0\",\"sv\":0,\"es_id\":\"cssek-0::TgcQVwLCU3kHIEj+o6s2CQ==:QTm2VyfemhN5owtHmVuMpik4hwFNz4gSpUQ9D9zlWxatYA==\"}}" + } else { + data["json"] = "{\"" + snap.Id + "\":{\"t\":1467469712.44128,\"replayed\":0,\"stack_id\":\"E437851F-2AA8-4C6C-AE11-5FB49FBF93C0\",\"sv\":0,\"es_id\":\"cssek-0::TgcQVwLCU3kHIEj+o6s2CQ==:QTm2VyfemhN5owtHmVuMpik4hwFNz4gSpUQ9D9zlWxatYA==\"}}" + } + res, err := GetHTTPResource("/bq/update_snaps", headers, data) + if err != nil { + return err + } + defer res.Body.Close() + return nil +} + +func ImageNameGenerator(seed string) (string, string) { + seedBytes := []byte(seed) + sha256Bytes := sha256.Sum256(seedBytes) + hash := hex.EncodeToString(sha256Bytes[:]) + folders := config.C.ContentDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + urls := config.C.ContentWebDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + if err := os.MkdirAll(folders, 0775); err != nil { + log.Fatalf("FAT Could not create ContentDirectory, error: %+v", err) + } + finalPath := folders + hash + finalUrl := urls + hash + return finalPath, finalUrl +} + +func RotateImage(snap Snap, data io.Reader, path string) (string, error) { + if snap.Movie == 1 { + ext := ".mp4" + blob, err := ioutil.ReadAll(data) + if err != nil { + return "", err + } + if len(blob) == 0 { + return "", errors.New("Empty response") + } + err = ioutil.WriteFile(path+ext, blob, 0644) + if err != nil { + return "", err + } + return ext, nil + } else { + ext := ".jpg" + image, err := imaging.Decode(data) + if err != nil { + return "", err + } + switch snap.Rotation { + case 1: + image = imaging.Rotate180(image) + break + case 2: + image = imaging.Rotate90(image) + break + case 3: + image = imaging.Rotate270(image) + break + default: + break + } + err = imaging.Save(image, path+ext) + if err != nil { + return "", err + } + return ext, nil + } +} diff --git a/snapchat/http.go b/snapchat/http.go new file mode 100644 index 0000000..d15bc12 --- /dev/null +++ b/snapchat/http.go @@ -0,0 +1,48 @@ +package snapchat + +import ( + "errors" + "log" + "net/http" + "net/url" + "strconv" + "strings" + + "git.1750studios.com/AniNite/SocialDragon/config" +) + +var sem = make(chan byte, 2) +var client = &http.Client{} + +func GetHTTPResource(u string, headers map[string]string, data map[string]string) (*http.Response, error) { + sem <- 1 + form := url.Values{} + for key, value := range data { + form.Set(key, value) + } + req, err := http.NewRequest("POST", config.C.Snapchat.ApiBase+u, strings.NewReader(form.Encode())) + if err != nil { + log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) + <-sem + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept-Locale", "de-DE") + req.Header.Set("User-Agent", config.C.Snapchat.UserAgent) + req.Header.Set("Accept-Language", "de;q=1, en;q=0.9, fr;q=0.8, zh-Hans;q=0.7, zh-Hant;q=0.6, ja;q=0.5") + for key, value := range headers { + req.Header.Set(key, value) + } + res, err := client.Do(req) + if err != nil { + <-sem + return nil, err + } + if res.StatusCode != 200 { + res.Body.Close() + <-sem + return nil, errors.New(strconv.Itoa(res.StatusCode)) + } + <-sem + return res, nil +} diff --git a/snapchat/main.go b/snapchat/main.go new file mode 100644 index 0000000..3bceda0 --- /dev/null +++ b/snapchat/main.go @@ -0,0 +1,57 @@ +package snapchat + +import ( + "log" + + "git.1750studios.com/AniNite/SocialDragon/database" +) + +func LoadNewSnaps() { + log.Print("Loading new Snaps...") + cons, err := GetConversations() + if err != nil { + log.Printf("Can't load snaps: %+v", err) + return + } + for _, con := range cons.ConversationsResponse { + for _, snap := range con.PendingReceivedSnaps { + var count int + if database.Db.Model(database.Item{}).Where("original_id = ?", snap.Id).Count(&count); count > 0 { + break + } + var US database.User + if database.Db.Model(database.User{}).Where("name = ? AND service = ?", snap.Username, database.Snapchat).First(&US).Count(&count); count == 0 { + US.DisplayName = snap.Username + US.Name = snap.Username + US.Service = database.Snapchat + US.Blocked = false + database.Db.Create(&US) + } + log.Printf("Found new Snap %s from %s", snap.Id, snap.Username) + uname, err := GetBlob(snap) + if err != nil { + log.Printf("Can't load snap %s: %+v", snap.Id, err) + return + } + log.Printf("Loaded snap %s, location %s!", snap.Id, uname) + var IT database.Item + IT.UserID = US.ID + IT.IsVideo = snap.Movie != 0 + IT.OriginalID = snap.Id + IT.Path = uname + IT.Service = database.Snapchat + if US.Blocked { + IT.State = database.Rejected + } else { + IT.State = database.Inbox + } + database.Db.Create(&IT) + err = MarkAsSeen(snap, false) + if err != nil { + log.Printf("Could not mark snap %s as seen: %+v", snap.Id, err) + return + } + } + } + log.Printf("Finished looking for new snaps.") +} diff --git a/socialdragon/main.go b/socialdragon/main.go new file mode 100644 index 0000000..ef9461d --- /dev/null +++ b/socialdragon/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "os" + "os/signal" + "syscall" + + "github.com/robfig/cron" + + "git.1750studios.com/AniNite/SocialDragon/config" + "git.1750studios.com/AniNite/SocialDragon/database" + "git.1750studios.com/AniNite/SocialDragon/instagram" + "git.1750studios.com/AniNite/SocialDragon/snapchat" + "git.1750studios.com/AniNite/SocialDragon/twitter" +) + +func main() { + config.LoadConfig(os.Getenv("HOME") + "/.socialdragon.toml") + database.InitDb() + + c := cron.New() + c.AddFunc("@every 30s", snapchat.LoadNewSnaps) + c.AddFunc("@every 30s", instagram.LoadNewInstas) + c.Start() + + go twitter.LoadNewTweets() + + ch := make(chan os.Signal) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + <-ch + + twitter.Stop() +} diff --git a/socialdragon/test.toml b/socialdragon/test.toml new file mode 100644 index 0000000..7cdcd00 --- /dev/null +++ b/socialdragon/test.toml @@ -0,0 +1,33 @@ +DatabaseConnection = "" +BindAddress = "" +AssetsDirectory = "" +TemplatesDirectory = "" +ContentDirectory = "" +ContentWebDirectory = "" + +[Snapchat] + ApiBase = "" + UserAgent = "" + UserName = "" + [Snapchat.GetConversations] + Uuid = "" + ClientAuthToken = "" + RequestToken = "" + Timestamp = "" + [Snapchat.GetBlob] + Uuid = "" + ClientAuthToken = "" + RequestToken = "" + Timestamp = "" + [Snapchat.MarkAsSeen] + Uuid = "" + ClientAuthToken = "" + RequestToken = "" + Timestamp = "" + +[Twitter] + ConsumerKey = "" + ConsumerSecret = "" + OAuthToken = "" + OAuthTokenSecret = "" + Filter = "" diff --git a/twitter/downloader.go b/twitter/downloader.go new file mode 100644 index 0000000..4ca5094 --- /dev/null +++ b/twitter/downloader.go @@ -0,0 +1,58 @@ +package twitter + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "io/ioutil" + "log" + "os" + + "github.com/disintegration/imaging" + + "git.1750studios.com/AniNite/SocialDragon/config" +) + +func MediaNameGenerator(seed string) (string, string) { + seedBytes := []byte(seed) + sha256Bytes := sha256.Sum256(seedBytes) + hash := hex.EncodeToString(sha256Bytes[:]) + folders := config.C.ContentDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + urls := config.C.ContentWebDirectory + "/" + hash[0:2] + "/" + hash[0:4] + "/" + if err := os.MkdirAll(folders, 0775); err != nil { + log.Fatalf("FAT Could not create ContentDirectory, error: %+v", err) + } + finalPath := folders + hash + finalUrl := urls + hash + return finalPath, finalUrl +} + +func DownloadMedia(data io.Reader, path string, video bool) (string, error) { + if video { + ext := ".mp4" + blob, err := ioutil.ReadAll(data) + if err != nil { + return "", err + } + if len(blob) == 0 { + return "", errors.New("Empty response") + } + err = ioutil.WriteFile(path+ext, blob, 0644) + if err != nil { + return "", err + } + return ext, nil + } else { + ext := ".jpg" + image, err := imaging.Decode(data) + if err != nil { + return "", err + } + err = imaging.Save(image, path+ext) + if err != nil { + return "", err + } + return ext, nil + } +} diff --git a/twitter/http.go b/twitter/http.go new file mode 100644 index 0000000..8223155 --- /dev/null +++ b/twitter/http.go @@ -0,0 +1,33 @@ +package twitter + +import ( + "errors" + "log" + "net/http" + "strconv" +) + +var sem = make(chan byte, 2) +var client = &http.Client{} + +func GetHTTPResource(u string) (*http.Response, error) { + sem <- 1 + req, err := http.NewRequest("GET", u, nil) + if err != nil { + log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) + <-sem + return nil, err + } + res, err := client.Do(req) + if err != nil { + <-sem + return nil, err + } + if res.StatusCode != 200 { + res.Body.Close() + <-sem + return nil, errors.New(strconv.Itoa(res.StatusCode)) + } + <-sem + return res, nil +} diff --git a/twitter/main.go b/twitter/main.go new file mode 100644 index 0000000..6452ead --- /dev/null +++ b/twitter/main.go @@ -0,0 +1,120 @@ +package twitter + +import ( + "log" + + "git.1750studios.com/AniNite/SocialDragon/config" + "git.1750studios.com/AniNite/SocialDragon/database" + "github.com/dghubble/go-twitter/twitter" + "github.com/dghubble/oauth1" +) + +var stream *twitter.Stream + +func LoadNewTweets() error { + log.Printf("Loading new tweets...") + conf := oauth1.NewConfig(config.C.Twitter.ConsumerKey, config.C.Twitter.ConsumerSecret) + token := oauth1.NewToken(config.C.Twitter.OAuthToken, config.C.Twitter.OAuthTokenSecret) + httpClient := conf.Client(oauth1.NoContext, token) + + client := twitter.NewClient(httpClient) + + params := &twitter.StreamFilterParams{ + Track: config.C.Twitter.Filter, + StallWarnings: twitter.Bool(false), + } + var err error + stream, err = client.Streams.Filter(params) + if err != nil { + return err + } + demux := twitter.NewSwitchDemux() + demux.Tweet = func(tweet *twitter.Tweet) { + if tweet.ExtendedEntities == nil || tweet.RetweetedStatus != nil { + return + } + var count int + var US database.User + if database.Db.Model(database.User{}).Where("name = ? AND service = ?", tweet.User.ScreenName, database.Twitter).First(&US).Count(&count); count == 0 { + US.DisplayName = tweet.User.Name + US.Name = tweet.User.ScreenName + US.Service = database.Twitter + US.Blocked = false + database.Db.Create(&US) + } + for _, media := range tweet.ExtendedEntities.Media { + if media.Type == "video" { + bitrate := 0 + index := 0 + for i, variant := range media.VideoInfo.Variants { + if variant.Bitrate > bitrate { + index = i + bitrate = variant.Bitrate + } + } + log.Printf("Found video in tweet %s from %s", tweet.IDStr, tweet.User.ScreenName) + name, uname := MediaNameGenerator(media.VideoInfo.Variants[index].URL) + res, err := GetHTTPResource(media.VideoInfo.Variants[index].URL) + if err != nil { + log.Printf("Can't load video from tweet %s: %+v", tweet.IDStr, err) + return + } + defer res.Body.Close() + ext, err := DownloadMedia(res.Body, name, true) + if err != nil { + log.Printf("Can't load video from tweet %s: %+v", tweet.IDStr, err) + return + } + log.Printf("Loaded tweet %s, location %s!", tweet.IDStr, uname+ext) + var IT database.Item + IT.UserID = US.ID + IT.Service = database.Twitter + if US.Blocked { + IT.State = database.Rejected + } else { + IT.State = database.Inbox + } + IT.IsVideo = true + IT.Path = uname + ext + IT.OriginalID = tweet.IDStr + database.Db.Create(&IT) + } else { + log.Printf("Found picture(s) in tweet %s from %s", tweet.IDStr, tweet.User.ScreenName) + name, uname := MediaNameGenerator(media.MediaURLHttps) + res, err := GetHTTPResource(media.MediaURLHttps) + if err != nil { + log.Printf("Can't load picture(s) from tweet %s: %+v", tweet.IDStr, err) + return + } + defer res.Body.Close() + ext, err := DownloadMedia(res.Body, name, false) + if err != nil { + log.Printf("Can't load picture(s) from tweet %s: %+v", tweet.IDStr, err) + return + } + log.Printf("Loaded tweet %s, location %s!", tweet.IDStr, uname+ext) + var IT database.Item + IT.UserID = US.ID + IT.Service = database.Twitter + if US.Blocked { + IT.State = database.Rejected + } else { + IT.State = database.Inbox + } + IT.IsVideo = false + IT.Path = uname + ext + IT.OriginalID = tweet.IDStr + database.Db.Create(&IT) + } + } + } + demux.HandleChan(stream.Messages) + log.Printf("Finished looking for new tweets.") + return nil +} + +func Stop() { + log.Printf("Stopping twitter stream...") + stream.Stop() + log.Printf("Stopped twitter stream.") +}