.*?
STORIES.*?.*?
\s*STORIES.*?(.*?)
.*?src="(.*?)" """
p = re.search( regex, html.decode('utf-8'), re_flags )
if p[3].endswith('lfi-plus.svg'):
# print('brämium gondend, exiting')
quit()
string2hash = p[0]
logger.debug(f"{p[0]}")
m.update(string2hash.encode('utf-8'))
hash = (m.hexdigest())
if not domain in meta_values: # first run
meta_values[domain] = { 'hash': '1' } # fake value
if not meta_values[domain]['hash'] == hash: # Unterschiedliche Hashes
logger.debug('unterschiedliche hashes')
logger.debug(f"search result {p[1]} {p[2]}")
# string2hash = f"""p[2]"""
m.update(string2hash.encode('utf-8'))
# hash = (m.hexdigest())
meta_values[domain] = { 'hash': hash,
'timestamp': timestamp - i,
'posttitle': p[2],
'posturl': p[1] }
q = {}
q[1] = p[1]
q[2] = p[2]
return (q, timestamp + i)
else:
logger.debug('gleiche hashes')
q = {}
q[1] = meta_values[domain]['posturl']
q[2] = meta_values[domain]['posttitle']
return (q, meta_values[domain]['timestamp'])
return (matchgroup, meta_values[domain]['timestamp'])
def picturesfromthezone(b, domain, i):
global meta_values
m = hashlib.sha256()
html = ""
ts = 0 # timestamp
with urllib.request.urlopen(b[1]) as response:
html = response.read()
string2hash = f"""html"""
m.update(string2hash.encode('utf-8'))
hash = (m.hexdigest())
if not domain in meta_values: # first run
logger.debug(domain)
meta_values[domain] = { 'hash': '1' } # fake value
if not meta_values[domain]['hash'] == hash: # Unterschiedliche Hashes
logger.debug('unterschiedliche hashes')
meta_values[domain] = { 'hash': hash,
'timestamp': timestamp - i,
'posttitle': '',
'posturl': b[1] }
q = {}
q[2] = ''
q[1] = b[1]
return (q, timestamp + i)
else:
logger.debug('gleiche hashes')
q = {}
q[1] = meta_values[domain]['posturl']
q[2] = meta_values[domain]['posttitle']
return (q, meta_values[domain]['timestamp'])
# return (b, meta_values[domain]['timestamp'])
def treat_special_domain(domain, b, i):
ts = 0
try:
if domain == 'www.orengrad.com':
(b, ts) = orengradcom(b, domain, i)
elif domain == 'lfi-online.de':
(b, ts) = lfionlinede(b, domain, i)
elif domain == 'photoplacegallery.com':
(b, ts) = photoplacegallery(b, domain, i)
elif domain == 'www.picturesfromthezone.com':
(b, ts) = picturesfromthezone(b, domain, i)
except:
pass
return (b, ts)
def read_spring2life_links():
#print('read_spring2life_links')
with urllib.request.urlopen(spring2life_links_url) as response:
html = response.read().decode('utf-8')
bloglist = reduce_lines(html)
regex = r"'blog-title'>\s*\s*(.*?)<\/a>(.*?)"
counter = 0
global list_of_blogs
for b in re.finditer(regex, bloglist, re.MULTILINE | re.DOTALL):
burl = b[1]
bdomain = re.sub( r"(https?://|/", bdomain)
if bdomain in alternative_blog_urls.keys():
burl = burl.replace(bdomain, alternative_blog_urls[bdomain])
# print(f"---->", burl)
btitle = b[2]
z = re.search(r".*?href='(.*?)'.*?>\s*(.*?)<.*?item-time'>\s*(.*?)\s*<", b[3], re.MULTILINE | re.DOTALL)
if z:
purl = z[1]
ptitle = z[2]
blogtimestamp = timestamp_to_epoch_secs(z[3], counter)
else:
(z, ts) = treat_special_domain(bdomain, b, counter)
blogtimestamp = ts
counter += 1
list_of_blogs[int(blogtimestamp)] = (f"""")
f.write(f"\t{list_of_blogs[t]}\n")
f.write("
")
logger = logging.getLogger(__name__)
# ------------------------------------------------------------- main ---
def main():
logging_config = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
# 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
'format': '[ %(lineno)s - %(funcName)25s() ] %(message)s'
},
},
'handlers': {
'default_handler': {'class': 'logging.StreamHandler',
'formatter': 'standard',
'level': loglevel },
# {
# 'class': 'logging.FileHandler',
# 'level': 'DEBUG',
# 'formatter': 'standard',
# 'filename': os.path.join('', 'application.log'),
# 'encoding': 'utf8'
# },
},
'loggers': {
'': {
'handlers': ['default_handler'],
'level': 'DEBUG',
'propagate': False
}
}
}
logging.config.dictConfig(logging_config)
read_value_hash()
read_spring2life_links()
output_list()
write_value_hash()
if __name__ == '__main__':
main()
# Local Variables:
# compile-command: "./blogsiread.py --log DEBUG"
# End: