diff --git a/gparser/feedparser.go b/gparser/feedparser.go index bab46a7..94598da 100644 --- a/gparser/feedparser.go +++ b/gparser/feedparser.go @@ -37,8 +37,8 @@ func ParseRSSFeeds() { log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err) continue } - defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) + res.Body.Close() if err != nil { log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err) continue @@ -64,6 +64,7 @@ func ParseRSSFeeds() { } wg.Wait() } + log.Printf("*** FINISHED PARSING RSS FEEDS ***") GenerateSitemap() } diff --git a/gparser/http.go b/gparser/http.go index 162f1c7..daa1654 100644 --- a/gparser/http.go +++ b/gparser/http.go @@ -8,12 +8,13 @@ import ( ) var sem = make(chan byte, 5) +var client = &http.Client{} + // Make an HTTP Get Request to u func GetHTTPResource(u string) (*http.Response, error) { sem <- 1 // Prepare HTTP Client, Cookie and Request - client := &http.Client {} cookie := &http.Cookie { Name : "POPUPCHECK", Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10), @@ -26,8 +27,8 @@ func GetHTTPResource(u string) (*http.Response, error) { req.Header.Add("Cache-Control", "max-age=0") req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12") if err != nil { - <- sem log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) + <- sem return nil, err } // Execute HTTP Request @@ -37,6 +38,7 @@ func GetHTTPResource(u string) (*http.Response, error) { return nil, err } if res.StatusCode == http.StatusNotFound { + res.Body.Close() <- sem return nil, nil } diff --git a/image/http.go b/image/http.go index 2d10de0..3a4085a 100644 --- a/image/http.go +++ b/image/http.go @@ -8,10 +8,13 @@ import ( "errors" ) +var sem = make(chan byte, 5) +var client = &http.Client {} + // Make an HTTP Get Request to u func GetHTTPResource(u string) (*http.Response, error) { + sem <- 1 // Prepare HTTP Client, Cookie and Request - client := &http.Client {} cookie := &http.Cookie { Name : "POPUPCHECK", Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10), @@ -25,15 +28,20 @@ func GetHTTPResource(u string) (*http.Response, error) { req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12") if err != nil { log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) + <- sem return nil, err } // Execute HTTP Request res, err := client.Do(req) if err != nil { + <- sem return nil, err } if res.StatusCode != 200 { + res.Body.Close() + <- sem return nil, errors.New(strconv.Itoa(res.StatusCode)) } + <- sem return res, nil } diff --git a/image/image.go b/image/image.go index 6ba68e6..0a35c2b 100644 --- a/image/image.go +++ b/image/image.go @@ -148,12 +148,13 @@ func ResizeImage(url string, out []string, size []string) (error) { if erro != nil { return erro } + defer f.Close() tfile := f.Name() + defer os.Remove(tfile) _, err = f.Write(blob) if err != nil { return err } - f.Close() for i := 0; i < len(out); i++ { cmd := exec.Command("convert", tfile, "-resize", size[i] + "^", "-quality", "70", "-gravity", "center", "-crop", size[i] + "+0+0", "+repage", out[i]) err = cmd.Run() @@ -161,6 +162,5 @@ func ResizeImage(url string, out []string, size []string) (error) { return err } } - os.Remove(tfile) return nil } diff --git a/youtube/youtube.go b/youtube/youtube.go index ad9b593..cf91f29 100644 --- a/youtube/youtube.go +++ b/youtube/youtube.go @@ -63,21 +63,28 @@ type YTResult struct { var videoEndpoint string = "https://www.googleapis.com/youtube/v3/videos" var channelEndpoint string = "https://www.googleapis.com/youtube/v3/channels" -var sem = make(chan byte, 10) +var sem = make(chan byte, 5) +var client = &http.Client {} // Make an HTTP Get Request to u func GetHTTPResource(u string) (*http.Response, error) { sem <- 1 // Prepare HTTP Client, Cookie and Request - res, err := http.Get(u) + req, err := http.NewRequest("GET", u, nil) if err != nil { - log.Printf("[WARN] HTTP - Request to %s failed: %s", u, err) + log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err) <- sem return nil, err } - if res.StatusCode == http.StatusNotFound { - log.Printf("[WARN] HTTP - Received 404 Not Found for: %s", u) + // Execute HTTP Request + res, err := client.Do(req) + if err != nil { + <- sem + return nil, err + } + if res.StatusCode == 404 { + res.Body.Close() <- sem return nil, nil }