spring2life-blogs-i-read/blogsiread.py

#!/usr/bin/python3

# 	$Id:$
#       $Revision:$
#       $Date:$
#       $Log:$


"""
* if yes
  * read the spring2life linklist on blogger,
  * special treatment for websites without feed
  * save list with timestamp into file
  * output list
"""
import json
import hashlib
import time
import datetime
import logging
import logging.config
import os
import os.path
import re
import socket
import time
import urllib.request

from pathlib import Path

spring2life_links_url = 'http://spring2life-links.blogspot.com/'
html_file             = '/home/springm/docker/wordpress-nginx/wordpress/wp-content/themes/twentytwentyone-child-spring2life/cronlinks.html'
database_file         = '/home/springm/docker/wordpress-nginx/blogsiread.json'
if socket.gethostname() == 'denkbrett' or socket.gethostname() == 'kudell':    # for development
    html_file             = 'cronlinks.html'
    database_file         = 'blogsiread.json'
timestamp             = int(time.time())
list_of_blogs         = {}
last_separator        = ''

def reduce_lines(html):
    lines = html.split('\n')
    i = 0
    j = 0
    found = 0
    bloglist = ''
    while i < len(lines):
        if lines[i] == "<ul id='BlogList1_blogs'>":
            found = 1
        if found == 1 and lines[i] == "</ul>":
            found = 0
            break
        if found == 1:
            # print(lines[i])
            bloglist = bloglist + lines[i]
        i = i + 1
    return(bloglist)

def timestamp_to_epoch_secs( time_text, i ):
    m = re.search(r"(\d+) (Sekunde|Minute|Stunde|Tag|Woche|Monat|Jahr)", time_text)
    if m:
        if m.group(2).startswith('Sekunde'):
            return timestamp - int(m.group(1)) - i
        if m.group(2).startswith('Minute'):
            return timestamp - int(m.group(1)) * 60 - i
        elif m.group(2).startswith('Stunde'):
            return timestamp - int(m.group(1)) * 3600 - i
        elif m.group(2).startswith('Tag'):
            return timestamp - int(m.group(1)) * 24 * 3600 - i
        elif m.group(2).startswith('Woche'):
            return timestamp - int(m.group(1)) * 7 * 24 * 3600 - i
        elif m.group(2).startswith('Monat'):
            return timestamp - int(m.group(1)) * 30 * 24 * 3600 - i
        elif m.group(2).startswith('Jahr'):
            return timestamp - int(m.group(1)) * 365 * 24 * 3600 - i
    # else:
    #     print(time_text)

def orengradcom(b, domain, i):
    global meta_values
    m = hashlib.sha256()
    html = ""
    ts = 0                      # timestamp
    url = 'https://www.orengrad.com/thingsseen/index.html'
    with urllib.request.urlopen(b[1]) as response:
        html = response.read()
        m.update(html)
        hash = (m.hexdigest())
        if not domain in meta_values: # first run
            meta_values[domain] = { 'hash': '1' } # fake value
        if not meta_values[domain]['hash'] == hash:      # Unterschiedliche Hashes
            logger.debug(f"unterschiedliche hashes")
            meta_values[domain]['hash'] = hash
            p = re.search('div id="bodycontent">.*<p><img src="pictures\/(.*?)\.(jpg)"', html.decode('utf-8'), re.MULTILINE | re.DOTALL )
            if p:
                logger.debug(f"match {p}")
                meta_values[domain] = { 'hash': hash,
                                        'timestamp': timestamp - i,
                                        'posttitle': '',
                                        'posturl': url }
                return (p, timestamp + i)
                # print(meta_values)
            else:
                pass
                #print('p is empty :(')
        else:
            pass
            #print('hashes are equal')
        return (b, meta_values[domain]['timestamp'])

def photoplacegallery(b, domain, i):
    # logger.debug(f"{domain}")
    global meta_values
    m = hashlib.sha256()
    html = ""
    ts = 0                      # timestamp
    url = 'https://photoplacegallery.com/online-juried-shows/'
    req = urllib.request.Request(b[1], None,
       { 'User-Agent' : 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
         'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
         'Accept-Language' : 'de,en-US;q=0.7,en;q=0.3',
         'Referer' : 'http://spring2life-links.blogspot.com/',
         'DNT' : '1',
         'Connection' : 'keep-alive',
         'Upgrade-Insecure-Requests' : '1',
         'Sec-Fetch-Dest' : 'document',
         'Sec-Fetch-Mode' : 'navigate',
         'Sec-Fetch-Site' : 'cross-site',
         'Pragma' : 'no-cache',
         'Cache-Control' : 'no-cache' })
    r = urllib.request.urlopen(req)
    with r as response:
        html = response.read()
        # hash only from content-relevant part of website
        subset = re.search('(<div class="main">.*?</div>\s*</div>\s*</div>)', html.decode('utf-8'),
                           re.MULTILINE | re.DOTALL )
        m.update(subset[1].encode('utf-8'))
        hash = (m.hexdigest())
        if not domain in meta_values: # first run
            meta_values[domain] = { 'hash': '1' } # fake value
        if not meta_values[domain]['hash'] == hash:      # Unterschiedliche Hashes
            # logger.debug(f"unterschiedliche hashes")
            meta_values[domain]['hash'] = hash
            p = re.search('<div class="widget-title"><a href="(/online-juried-shows/[^"]*?)" class="h3">([^<]*?)</a></div>', html.decode('utf-8'), re.MULTILINE | re.DOTALL )
            if p:
                # logger.debug(f"re search erfolgreich: {p[1]} {p[2]}")
                meta_values[domain] = { 'hash': hash,
                                        'timestamp': timestamp - i,
                                        'posttitle': p[2],
                                        'posturl': f"https://{domain}{p[1]}" }
                q = {}
                q[1] = f"https://{domain}{p[1]}"
                q[2] = p[2]
                return (q, timestamp + i)
                # print(meta_values)
            else:
                pass
                #print('p is empty :(')
        else:
            # logger.debug(f"gleiche hashes")
            q = {}
            q[1] = meta_values[domain]['posturl']
            q[2] = meta_values[domain]['posttitle']
            return (q, meta_values[domain]['timestamp'])
            #print('hashes are equal')
        return (b, meta_values[domain]['timestamp'])

def lfionlinede(b, domain, i):
    global meta_values
    m = hashlib.sha256()
    html = ""
    ts = 0                      # timestamp
    url = 'https://lfi-online.de/ceemes/de/blog/'
    # logger.debug(f"{b[1]}")
    with urllib.request.urlopen(b[1]) as response:
        html = response.read()
        m.update(html)
        hash = (m.hexdigest())
        if not domain in meta_values: # first run
            meta_values[domain] = { 'hash': '1' } # fake value
        if not meta_values[domain]['hash'] == hash:      # Unterschiedliche Hashes
            meta_values[domain]['hash'] = hash
            p = re.search('titlebox30 cu-pointer[\'"] onclick=[\'"]window.location\s*=\s*[\'"](https://.*?)[\'"][\'"]>\s*<h1.*?>(.*?)</h1', html.decode('utf-8'), re.MULTILINE | re.DOTALL)
            if p:
                logger.debug(f"search result {p[1]} {p[2]}")
                meta_values[domain] = { 'hash': hash,
                                        'timestamp': timestamp - i,
                                        'posttitle': p[2],
                                        'posturl':   p[1] }
                q = {}
                q[1] = p[1]
                q[2] = p[2]
                return (p, timestamp + i)
                # print(meta_values)
            else:
                pass
                #print('p is empty :(')
        else:
            logger.debug('hashes are equal')
            q = {}
            q[1] = meta_values[domain]['posturl']
            q[2] = meta_values[domain]['posttitle']
            return (q, meta_values[domain]['timestamp'])
        return (b, meta_values[domain]['timestamp'])

def treat_special_domain(domain, b, i):
    ts = 0
    if domain == 'www.orengrad.com':
        (b, ts)  = orengradcom(b, domain, i)
    # elif domain == 'jims-ramblings.blogspot.com':
    #     print(f"special: {domain}")
    elif domain == 'lfi-online.de':
        (b, ts)  = lfionlinede(b, domain, i)
    elif domain == 'photoplacegallery.com':
        (b, ts)  = photoplacegallery(b, domain, i)
        logger.debug(f"{b[1]} {b[2]}")
    # elif domain == 'www.picturesfromthezone.com':
    #     print(f"special: {domain}")
    return (b, ts)

def read_spring2life_links():
    #print('read_spring2life_links')
    with urllib.request.urlopen(spring2life_links_url) as response:
        html = response.read().decode('utf-8')
        bloglist = reduce_lines(html)
    regex = r"'blog-title'>\s*<a href='(.*?)'.*?>\s*(.*?)<\/a>(.*?)<div style='clear: both;'>"
    counter = 0
    global list_of_blogs
    for b in re.finditer(regex, bloglist, re.MULTILINE | re.DOTALL):
        burl = b[1]
        bdomain = re.sub( r"(https?://|/<?.*?$)", "", burl)
        btitle = b[2]
        z = re.search(r".*?href='(.*?)'.*?>\s*(.*?)<.*?item-time'>\s*(.*?)\s*<", b[3], re.MULTILINE | re.DOTALL)
        if z:
            purl = z[1]
            ptitle = z[2]
            blogtimestamp = timestamp_to_epoch_secs(z[3], counter)
        else:
            (z, ts) = treat_special_domain(bdomain, b, counter)
            blogtimestamp = ts
            if bdomain == 'lfi-online.de':
                logger.debug(f"into list: \n{b[1]} // {b[2]}\n{z[1]} // {z[2]}")
        counter += 1
        list_of_blogs[int(blogtimestamp)] = (f"""<li><a href='{b[1]}' target='_blank'>{b[2]}</a>"""
                                             f"""&nbsp;//&nbsp;<a href='{z[1]}' target='_blank'>{z[2]}</a></li>""")

def read_value_hash():
    global meta_values
    try:
        f = open(database_file, 'r')
        meta_values = json.loads(f.read())
        # meta_values['lfi-online.de']['hash'] = 'abc' # for testing, set false hash
        # print(meta_values)
    except:
        meta_values = {}

def write_value_hash():
    f = open(database_file, 'w+')
    f.write(json.dumps(meta_values))

def separator(t):
    global last_separator
    # print(f"{timestamp - t} -- {last_separator}")
    if ( timestamp - t ) > 10368000:
        if not last_separator == "From medieval times": # 24*30*24*600
            last_separator = "From medieval times"
            return last_separator
    elif ( timestamp - t ) > 2592000:
        if not last_separator == "Quite old": # 6*30*24*600
            last_separator = "Quite old"
            return last_separator
    elif ( timestamp - t ) > 432000:
        if not last_separator == "Less then a month": # 30*24*600
            last_separator = "Less then a month"
            return last_separator
    elif ( timestamp - t ) > 100800:
        if not last_separator == "Less then a week": # 7*24*600
            last_separator = "Less then a week"
            return last_separator
    elif ( timestamp - t ) > 86400: 
        if not last_separator == "A day and older": # 24*600
            last_separator = "A day and older"
            return last_separator
    elif ( timestamp - t ) < 86400: 
        if not last_separator == "Hot from the Blogosphere": # 24*600
            last_separator = "Hot from the Blogosphere"
            return last_separator
    return False


def output_list():
    # print(timestamp)
    with open(html_file, "w") as f:
        # f.write( f"{blogtimestamp};{bdomain};{burl};{btitle};{purl};{ptitle}\n" )
        firstsep = True
        for t in sorted(list_of_blogs, reverse=True):
            sep = separator(t)
            if sep:
                if not firstsep:
                    f.write("</ul>")
                else:
                    firstsep = False
                f.write(f"<li style='font-weight: bold;'>{sep}</li>\n<ul>")
            f.write(f"\t{list_of_blogs[t]}\n")
        f.write("</ul>")


logger = logging.getLogger(__name__)
# ------------------------------------------------------------- main ---
def main():
    logging_config = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'standard': {
                # 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
                'format': '[%(lineno)s - %(funcName)25s() ] %(message)s'
            },
        },
        'handlers': {
            'default_handler': {'class': 'logging.StreamHandler',
                                'formatter': 'standard',
                                'level': logging.WARN },
            # {
            #     'class': 'logging.FileHandler',
            #     'level': 'DEBUG',
            #     'formatter': 'standard',
            #     'filename': os.path.join('', 'application.log'),
            #     'encoding': 'utf8'
            # },
        },
        'loggers': {
            '': {
                'handlers': ['default_handler'],
                'level': 'DEBUG',
                'propagate': False
            }
        }
    }
    logging.config.dictConfig(logging_config)
    
    read_value_hash()
    read_spring2life_links()
    output_list()
    write_value_hash()

if __name__ == '__main__':
    main()
    
# Local Variables:
# compile-command: "./blogsiread.py --log DEBUG"
# End: