Fixing too many open file descriptors (closing sockets after using them)
This commit is contained in:
parent
b8e0e74f73
commit
f52708e7d3
|
@ -37,8 +37,8 @@ func ParseRSSFeeds() {
|
||||||
log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err)
|
log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
defer res.Body.Close()
|
|
||||||
body, err := ioutil.ReadAll(res.Body)
|
body, err := ioutil.ReadAll(res.Body)
|
||||||
|
res.Body.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err)
|
log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err)
|
||||||
continue
|
continue
|
||||||
|
@ -64,6 +64,7 @@ func ParseRSSFeeds() {
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
log.Printf("*** FINISHED PARSING RSS FEEDS ***")
|
||||||
GenerateSitemap()
|
GenerateSitemap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,12 +8,13 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var sem = make(chan byte, 5)
|
var sem = make(chan byte, 5)
|
||||||
|
var client = &http.Client{}
|
||||||
|
|
||||||
|
|
||||||
// Make an HTTP Get Request to u
|
// Make an HTTP Get Request to u
|
||||||
func GetHTTPResource(u string) (*http.Response, error) {
|
func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
sem <- 1
|
sem <- 1
|
||||||
// Prepare HTTP Client, Cookie and Request
|
// Prepare HTTP Client, Cookie and Request
|
||||||
client := &http.Client {}
|
|
||||||
cookie := &http.Cookie {
|
cookie := &http.Cookie {
|
||||||
Name : "POPUPCHECK",
|
Name : "POPUPCHECK",
|
||||||
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
||||||
|
@ -26,8 +27,8 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
req.Header.Add("Cache-Control", "max-age=0")
|
req.Header.Add("Cache-Control", "max-age=0")
|
||||||
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
<- sem
|
|
||||||
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||||
|
<- sem
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Execute HTTP Request
|
// Execute HTTP Request
|
||||||
|
@ -37,6 +38,7 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if res.StatusCode == http.StatusNotFound {
|
if res.StatusCode == http.StatusNotFound {
|
||||||
|
res.Body.Close()
|
||||||
<- sem
|
<- sem
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,10 +8,13 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var sem = make(chan byte, 5)
|
||||||
|
var client = &http.Client {}
|
||||||
|
|
||||||
// Make an HTTP Get Request to u
|
// Make an HTTP Get Request to u
|
||||||
func GetHTTPResource(u string) (*http.Response, error) {
|
func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
|
sem <- 1
|
||||||
// Prepare HTTP Client, Cookie and Request
|
// Prepare HTTP Client, Cookie and Request
|
||||||
client := &http.Client {}
|
|
||||||
cookie := &http.Cookie {
|
cookie := &http.Cookie {
|
||||||
Name : "POPUPCHECK",
|
Name : "POPUPCHECK",
|
||||||
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
||||||
|
@ -25,15 +28,20 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||||
|
<- sem
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Execute HTTP Request
|
// Execute HTTP Request
|
||||||
res, err := client.Do(req)
|
res, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
<- sem
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if res.StatusCode != 200 {
|
if res.StatusCode != 200 {
|
||||||
|
res.Body.Close()
|
||||||
|
<- sem
|
||||||
return nil, errors.New(strconv.Itoa(res.StatusCode))
|
return nil, errors.New(strconv.Itoa(res.StatusCode))
|
||||||
}
|
}
|
||||||
|
<- sem
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -148,12 +148,13 @@ func ResizeImage(url string, out []string, size []string) (error) {
|
||||||
if erro != nil {
|
if erro != nil {
|
||||||
return erro
|
return erro
|
||||||
}
|
}
|
||||||
|
defer f.Close()
|
||||||
tfile := f.Name()
|
tfile := f.Name()
|
||||||
|
defer os.Remove(tfile)
|
||||||
_, err = f.Write(blob)
|
_, err = f.Write(blob)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
f.Close()
|
|
||||||
for i := 0; i < len(out); i++ {
|
for i := 0; i < len(out); i++ {
|
||||||
cmd := exec.Command("convert", tfile, "-resize", size[i] + "^", "-quality", "70", "-gravity", "center", "-crop", size[i] + "+0+0", "+repage", out[i])
|
cmd := exec.Command("convert", tfile, "-resize", size[i] + "^", "-quality", "70", "-gravity", "center", "-crop", size[i] + "+0+0", "+repage", out[i])
|
||||||
err = cmd.Run()
|
err = cmd.Run()
|
||||||
|
@ -161,6 +162,5 @@ func ResizeImage(url string, out []string, size []string) (error) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
os.Remove(tfile)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,21 +63,28 @@ type YTResult struct {
|
||||||
|
|
||||||
var videoEndpoint string = "https://www.googleapis.com/youtube/v3/videos"
|
var videoEndpoint string = "https://www.googleapis.com/youtube/v3/videos"
|
||||||
var channelEndpoint string = "https://www.googleapis.com/youtube/v3/channels"
|
var channelEndpoint string = "https://www.googleapis.com/youtube/v3/channels"
|
||||||
var sem = make(chan byte, 10)
|
|
||||||
|
|
||||||
|
var sem = make(chan byte, 5)
|
||||||
|
var client = &http.Client {}
|
||||||
|
|
||||||
// Make an HTTP Get Request to u
|
// Make an HTTP Get Request to u
|
||||||
func GetHTTPResource(u string) (*http.Response, error) {
|
func GetHTTPResource(u string) (*http.Response, error) {
|
||||||
sem <- 1
|
sem <- 1
|
||||||
// Prepare HTTP Client, Cookie and Request
|
// Prepare HTTP Client, Cookie and Request
|
||||||
res, err := http.Get(u)
|
req, err := http.NewRequest("GET", u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[WARN] HTTP - Request to %s failed: %s", u, err)
|
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||||
<- sem
|
<- sem
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if res.StatusCode == http.StatusNotFound {
|
// Execute HTTP Request
|
||||||
log.Printf("[WARN] HTTP - Received 404 Not Found for: %s", u)
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
<- sem
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if res.StatusCode == 404 {
|
||||||
|
res.Body.Close()
|
||||||
<- sem
|
<- sem
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue