Fixing too many open file descriptors (closing sockets after using them)

gparser/feedparser.go

@@ -37,8 +37,8 @@ func ParseRSSFeeds() {
             log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err)
             continue
         }
-        defer res.Body.Close()
         body, err := ioutil.ReadAll(res.Body)
+        res.Body.Close()
         if err != nil {
             log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err)
             continue
@@ -64,6 +64,7 @@ func ParseRSSFeeds() {
         }
         wg.Wait()
     }
+    log.Printf("*** FINISHED PARSING RSS FEEDS ***")
     GenerateSitemap()
 }
 

gparser/http.go

@@ -8,12 +8,13 @@ import (
 )
 
 var sem = make(chan byte, 5)
+var client = &http.Client{}
+
 
 // Make an HTTP Get Request to u
 func GetHTTPResource(u string) (*http.Response, error) {
     sem <- 1
     // Prepare HTTP Client, Cookie and Request
-    client := &http.Client {}
     cookie := &http.Cookie {
         Name : "POPUPCHECK",
         Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
@@ -26,8 +27,8 @@ func GetHTTPResource(u string) (*http.Response, error) {
     req.Header.Add("Cache-Control", "max-age=0")
     req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
     if err != nil {
-        <- sem
         log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
+        <- sem
         return nil, err
     }
     // Execute HTTP Request
@@ -37,6 +38,7 @@ func GetHTTPResource(u string) (*http.Response, error) {
         return nil, err
     }
     if res.StatusCode == http.StatusNotFound {
+        res.Body.Close()
         <- sem
         return nil, nil
     }

image/http.go

@@ -8,10 +8,13 @@ import (
     "errors"
 )
 
+var sem = make(chan byte, 5)
+var client = &http.Client {}
+
 // Make an HTTP Get Request to u
 func GetHTTPResource(u string) (*http.Response, error) {
+    sem <- 1
     // Prepare HTTP Client, Cookie and Request
-    client := &http.Client {}
     cookie := &http.Cookie {
         Name : "POPUPCHECK",
         Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
@@ -25,15 +28,20 @@ func GetHTTPResource(u string) (*http.Response, error) {
     req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
     if err != nil {
         log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
+        <- sem
         return nil, err
     }
     // Execute HTTP Request
     res, err := client.Do(req)
     if err != nil {
+        <- sem
         return nil, err
     }
     if res.StatusCode != 200 {
+        res.Body.Close()
+        <- sem
         return nil, errors.New(strconv.Itoa(res.StatusCode))
     }
+    <- sem
     return res, nil
 }

image/image.go

@@ -148,12 +148,13 @@ func ResizeImage(url string, out []string, size []string) (error) {
     if erro != nil {
         return erro
     }
+    defer f.Close()
     tfile := f.Name()
+    defer os.Remove(tfile)
     _, err = f.Write(blob)
     if err != nil {
         return err
     }
-    f.Close()
     for i := 0; i < len(out); i++ {
         cmd := exec.Command("convert", tfile, "-resize", size[i] + "^", "-quality", "70", "-gravity", "center", "-crop", size[i] + "+0+0", "+repage", out[i])
         err = cmd.Run()
@@ -161,6 +162,5 @@ func ResizeImage(url string, out []string, size []string) (error) {
             return err
         }
     }
-    os.Remove(tfile)
     return nil
 }

youtube/youtube.go

@@ -63,21 +63,28 @@ type YTResult struct {
 
 var videoEndpoint string = "https://www.googleapis.com/youtube/v3/videos"
 var channelEndpoint string = "https://www.googleapis.com/youtube/v3/channels"
-var sem = make(chan byte, 10)
 
+var sem = make(chan byte, 5)
+var client = &http.Client {}
 
 // Make an HTTP Get Request to u
 func GetHTTPResource(u string) (*http.Response, error) {
     sem <- 1
     // Prepare HTTP Client, Cookie and Request
-    res, err := http.Get(u)
+    req, err := http.NewRequest("GET", u, nil)
     if err != nil {
-        log.Printf("[WARN] HTTP - Request to %s failed: %s", u, err)
+        log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
         <- sem
         return nil, err
     }
-    if res.StatusCode == http.StatusNotFound {
+    // Execute HTTP Request
-        log.Printf("[WARN] HTTP - Received 404 Not Found for: %s", u)
+    res, err := client.Do(req)
+    if err != nil {
+        <- sem
+        return nil, err
+    }
+    if res.StatusCode == 404 {
+        res.Body.Close()
         <- sem
         return nil, nil
     }