|
@@ -41,6 +41,7 @@ results = {}
|
|
|
|
|
|
|
|
def get_feed_content(url):
|
|
def get_feed_content(url):
|
|
|
count = 0
|
|
count = 0
|
|
|
|
|
+ feed = ''
|
|
|
while count <= 3:
|
|
while count <= 3:
|
|
|
count += 1
|
|
count += 1
|
|
|
#if True:
|
|
#if True:
|
|
@@ -184,9 +185,9 @@ def examine_url(url):
|
|
|
loaded_cookies = md5_sums[url]['cookies']
|
|
loaded_cookies = md5_sums[url]['cookies']
|
|
|
except:
|
|
except:
|
|
|
loaded_cookies = {}
|
|
loaded_cookies = {}
|
|
|
- response = requests.get(url, cookies=loaded_cookies, timeout=feed_timeout)
|
|
|
|
|
#if True:
|
|
#if True:
|
|
|
try:
|
|
try:
|
|
|
|
|
+ response = requests.get(url, cookies=loaded_cookies, timeout=feed_timeout)
|
|
|
saved_cookies = requests.utils.dict_from_cookiejar(response.cookies)
|
|
saved_cookies = requests.utils.dict_from_cookiejar(response.cookies)
|
|
|
cookies_json = json.dumps(saved_cookies, indent=4)
|
|
cookies_json = json.dumps(saved_cookies, indent=4)
|
|
|
md5_sums[url]['cookies'] = saved_cookies
|
|
md5_sums[url]['cookies'] = saved_cookies
|
|
@@ -201,7 +202,7 @@ def examine_url(url):
|
|
|
else:
|
|
else:
|
|
|
(md5, post_title, post_url, last_update) = examine_generic_website(soup, url, md5)
|
|
(md5, post_title, post_url, last_update) = examine_generic_website(soup, url, md5)
|
|
|
except:
|
|
except:
|
|
|
- pass
|
|
|
|
|
|
|
+ logger.warn(f'Error in {url}')
|
|
|
return md5, post_title, post_url, last_update
|
|
return md5, post_title, post_url, last_update
|
|
|
|
|
|
|
|
def needs_update(url):
|
|
def needs_update(url):
|