#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020-02-04 14:34
# @Author  : Lemon
# @File    : _parse_html.py
# @Software: PyCharm
import base64
import json
import re
import time
from urllib import parse

import bs4
import demjson
import furl
from bs4 import BeautifulSoup
from lxml import etree


def friend_unit(p: dict):
    next_url = None
    result = list()
    html = _unit_html(p)
    bs = BeautifulSoup(html, "html.parser")

    if 'objectListItem' in html:
        for li in bs.select('.objectListItem'):
            a_label = li.find_all('a', limit=2)
            try:
                src = a_label[0].find('img').attrs['src']
            except:
                src = None
            data = a_label[1].attrs
            thread_id = re.findall(r'user.php\?id=(\d+)&', data['data-hovercard'])[0]
            title = data['title']
            if thread_id and title:
                result.append({'fbid': thread_id, 'name': title, 'src': src})
    elif 'friendBrowserListUnit' in html:
        for li in bs.select('.friendBrowserListUnit'):
            unit = li.find('div', class_='friendBrowserUnit')
            thread_id = unit.find('input', class_='friendBrowserID').attrs['value']
            try:
                img_attrs=li.find('img').attrs
                title=img_attrs['aria-label']
                src=img_attrs['src']
                result.append({'fbid': thread_id, 'name': title, 'src': src})
            except:
                result.append({'fbid': thread_id, 'name': None, 'src': None})
    elif 'friendBrowserContent' in html:
        for li in bs.select('.friendBrowserContent'):
            try:
                src = li.parent.parent.parent.parent.find('img').attrs['src']
            except:
                src = None
            data = li.find_all('a', limit=2)[0].attrs
            if 'data-hovercard' in data:
                thread_id = re.findall(r'user.php\?id=(\d+)&', data['data-hovercard'])[0]
                title = data['title']
                if thread_id and title:
                    result.append({'fbid': thread_id, 'name': title, 'src': src})

    if 'uiMorePagerPrimary' in html:
        atag = bs.find('a', attrs={'class': 'uiMorePagerPrimary'})
        next_url = atag.get('ajaxify')
    return result, next_url


def _unit_html(p):
    try:
        html = get_domops_3(p)
        assert html
    except:
        html = p.get('payload', {}).get('results', {}).get('__html', '')
    return html


def find_input_fields_with_pc(html):
    b = bs4.BeautifulSoup(html, "html.parser")
    login_form = b.select("._featuredLogin__formContainer")[0]
    return login_form.find_all("input")


def get_publickey_id(text):
    try:
        pub = re.findall(r'"pubKey":({.*?})', text)[0]
        keys = json.loads(pub)
        return keys['publicKey'], keys['keyId']
    except:
        return None, None


def show_home_page(html):
    b = bs4.BeautifulSoup(html, "html.parser")
    return b


def get_domops_3(res):
    html = ""
    try:
        for domops in res.get('domops'):
            for x in domops:
                if isinstance(x, dict):
                    html = x['__html']
                    break
            if html:
                break
    except:
        pass
    return html


def get_overview_text(res):
    html = get_domops_3(res)
    msgs = []
    if html:
        FILTER = re.compile(r"Edit the places you've lived|Edit your work|Edit your education")
        b = bs4.BeautifulSoup(html, "html.parser")
        lis = b.find_all("div", class_="clearfix")
        for i, item in enumerate(lis):
            if i:
                text = FILTER.sub(lambda x: "", item.text)
                msgs.append(text)
    return "\n".join(msgs)


def get_current_city(res):
    html = get_domops_3(res)
    city_text = city_id = None
    if html:
        try:
            b = bs4.BeautifulSoup(html, 'html.parser')
            city = b.find('li', id='current_city')
            a_label = city.find('a')
            if a_label:
                url = "https://www.facebook.com" + a_label.attrs['data-hovercard']
                city_text = a_label.text
                city_id = re.findall(r"page.php\?id=(\d+)&", url)[0]
        except:
            pass
    return city_text, city_id


def get_user_info(b):
    pattern = re.compile(r"viewer_actor:(.*?)comment_count", re.MULTILINE | re.DOTALL)
    script = b.find("script", text=pattern)
    if not script:
        a_lable = b.find('a', attrs={'data-gt': '{"chrome_nav_item":"timeline_chrome"}'})
        name = a_lable.text
        url = a_lable.attrs.get('href')
        img_data = a_lable.find('img').attrs
        image = img_data.get('src')
        id = re.search("_header_(\d+)", a_lable.find('img').attrs.get('id')).group(1)
    else:
        info = pattern.search(script.text).group()
        id = re.findall(r'id:"(.*?)"', info)[0]
        name = re.findall(r',name:"(.*?)"', info)[0]
        url = re.findall(r'url:"(.*?)"', info)[0]
        image = re.findall(r'profile_picture_depth_0.*?uri:"(.*?)"', info)[0]
    try:
        r = re.compile('/p\d+x\d+/(.*?)\?')
        iname = r.findall(image)[0]
        pattern = re.compile('src="(https.*?/p\d{3,}x\d{3,}/%s.*?)"' % (iname), re.MULTILINE | re.DOTALL)
        elem = b.find('div', class_='hidden_elem', string=pattern)
        image = re.sub('&', lambda x: "&", pattern.findall(elem.string)[0])
    except:
        pass
    return id, name, url, image


def get_all_raw_id(text):
    valid = []
    b = bs4.BeautifulSoup(text, 'html.parser')
    if 'raw_id\\\\":' in text:
        pattern = re.compile(r'raw_id\\\\":(\d+),')
        elem = b.find_all('div', class_='hidden_elem', string=pattern)
        for e in elem:
            # key = pattern.findall(e.string)
            # fbids.extend(key)
            btn_div = bs4.BeautifulSoup(e.string, 'html.parser')
            btn_divs = btn_div.find_all('div', class_='FriendButton')
            for div in btn_divs:
                btn = div.find(lambda tag: tag.name == 'button' and tag.has_attr('data-profileid'))
                valid.append(btn.get('data-profileid'))
        valid = list(set(valid))
    else:
        # r = re.compile(r'\\"raw_id\\":(\d+),')
        # key = r.findall(text)
        btn_div = bs4.BeautifulSoup(text, 'html.parser')
        btn_divs = btn_div.find_all('div', class_='FriendButton')
        for div in btn_divs:
            btn = div.find(lambda tag: tag.name == 'button' and tag.has_attr('data-profileid'))
            valid.append(btn.get('data-profileid'))
        valid = list(set(valid))

    return valid


def get_all_unit_id(text):
    reg = re.compile(r'%22unit_id_result_id%22%3A%22(.*?)%22')
    split = re.compile(r'S:_I(\d+):(\d+)')
    post_ids = reg.findall(text)
    result = []
    for postid in post_ids:
        try:
            sid = base64.b64decode(parse.unquote(postid)).decode()
            result.append('_'.join(split.search(sid).groups(1)))
        except:
            pass
    return result

def get_hidden_input(res):
    text = get_domops_3(res)
    b = bs4.BeautifulSoup(text, 'html.parser')
    ins = [x.attrs for x in b.find_all('input', type='hidden')]

    args = {}
    for x in ins:
        if 'name' in x and 'value' in x:
            args.update({x['name']: x['value']})
    return args


def get_div_text(html) -> str:
    b = bs4.BeautifulSoup(html, 'html.parser')
    return b.text


def get_friend_div(text):
    if 'TimelineAppCollectionListRenderer' in text:
        b = bs4.BeautifulSoup(text, 'html.parser')
        script = b.find('script', string=re.compile(r'TimelineAppCollectionListRenderer'))
        a = '{}'
        if ',null_state_msg' in script.string:
            a = '{%s}' % \
                re.findall(r'TimelineAppCollectionListRenderer",collection:\{(.*?),null_state_msg', script.string)[0]
        else:
            a = re.findall(r'TimelineAppCollectionListRenderer",collection:(\{.*?\}\}\})', script.string)[0]
        res = demjson.decode(a)
        return res, None
    elif 'friend_list_item' in text:
        data = {'items': {'edges': [], 'count': 0}}
        b = bs4.BeautifulSoup(text, 'html.parser')
        elem = b.find('div', class_='hidden_elem', string=re.compile(r'data-testid="friend_list_item"'))
        if elem:
            elem = bs4.BeautifulSoup(elem.string, 'html.parser')
            divs = elem.find_all('div', {"data-testid": "friend_list_item"})
            first_page = True
        else:
            first_page = False
            divs = b.find_all('li')

        for dd in divs:
            node = dict()
            item = dict()
            try:
                a_data = dd.find('a').attrs
                img_data = dd.find('img').attrs

                url = a_data.get('href')
                fbid = re.findall(r'user.php\?id=(\d+)&', a_data.get('data-hovercard', ""))[0]
                image = img_data.get('src')
                name = img_data.get('aria-label')
                item['image'] = {'uri': image}
                item['title'] = {'text': name}
                item['can_friend'] = dd.find('button', class_='FriendRequestAdd') is not None
                item['node'] = {'id': fbid, 'url': url}
                node['node'] = item
                data['items']['edges'].append(node)
            except:
                pass
        if first_page:
            script = b.find('script', string=re.compile('"MedleyPageletRequestData"'))
            page = re.compile(r'"MedleyPageletRequestData","set",\[\],\[(.*?)\]')
            ext_data = demjson.decode(page.search(script.string).group(1))

            load = re.compile('\["TimelineAppCollection","enableContentLoader".*?\]\]')
            script_126 = b.find('script', string=load)
            if script_126:
                tttt = load.search(script_126.string).group()
                ext_data.update({
                    'cursor': re.search(r'"([A-Za-z0-9-_]{50,200})"', tttt).group(1),
                    'collection_token': re.search(r'"pagelet_timeline_app_collection_(.*?)"', tttt).group(1),
                })
                data['items']['has_next_page'] = True
            else:
                data['items']['has_next_page'] = False

            count_item = re.search(r'AllFriendsAppCollectionPagelet".*?,tab_count:(\d+),', text)
            data['items']['count'] = count_item and int(count_item.group(1)) or 0

            ext_data['count'] = data['items']['count']
            data['items']['ext_data'] = ext_data
        else:
            data['items']['ext_data'] = "update"

        return data, None
    elif 'uiInterstitialContent' in text:
        msg = 'We limit how often you can post, comment or do other things in a given amount of time in order to help protect the community from spam. You can try again later. '
        return {}, msg
    else:
        return {}, 'No friends to show'


def get_pagelet_info(res):
    y = {}
    for x in res['jsmods']['require']:
        if isinstance(x, list) and x[0] == 'TimelineAppCollection':
            y = x
            break
    try:
        cursor = y[3][2]
    except:
        cursor = None
    return cursor


def get_moment_info(payload):
    html = payload['actions'][0]['html']
    b = bs4.BeautifulSoup(html, 'html.parser')
    items = []
    try:
        for article in b.find_all('article'):
            header = article.find('header')
            text = header.next_sibling
            content = text.text
            imgs = text.next_sibling
            title = header.find('h3').text
            href = text.find('a').get('href')
            time = header.find('abbr').text
            story_fbid = re.findall(r'story_fbid=(\d+)', href)
            img_urls, video_urls = [], []
            if imgs:
                img_a = imgs.find_all('a')
                for x in img_a:
                    i = x.find('i')
                    if not i: continue
                    img_style = i.get('style')
                    if not img_style:
                        continue
                    encode_url = re.findall(r"url\('(.*?)'\)", img_style)[0]
                    decode_url = parse.unquote(re.sub('\\\\[0-9A-Fa-f]{2} ',
                                                      lambda x: x.group().replace('\\', '%').strip(),
                                                      encode_url))
                    img_urls.append(decode_url)

                video_div = imgs.find('div', attrs={'data-store': re.compile('video')})
                if video_div:
                    video_src = video_div.get('data-store')
                    video_src = json.loads(video_src)['src']
                    video_urls.append(video_src)

                    preview = video_div.find('i')
                    if not preview: continue
                    img_style = preview.get('style')
                    encode_url = re.findall(r"url\('(.*?)'\)", img_style)[0]
                    decode_url = parse.unquote(re.sub('\\\\[0-9A-Fa-f]{2} ',
                                                      lambda x: x.group().replace('\\', '%').strip(),
                                                      encode_url))
                    img_urls.append(decode_url)

            storedata = header.find('div', attrs={'data-store': re.compile('feed')})
            if storedata:
                store = json.loads(storedata.get('data-store'))
                if 'feedobjectsIdentifiers' in store:
                    # S:_I100048434771169:104943314463494:0
                    story_fbid = store.get('feedobjectsIdentifiers')
                elif 'saveInfo' in store:
                    # S:_I100045812690193:136820304521725
                    story_fbid = store['saveInfo']['logData']['story_id']

            items.append({
                'story_fbid': story_fbid,
                'title': title,
                'content': content,
                'imgs': img_urls,
                'videos': video_urls,
                'time': time,
            })
    except Exception as e:
        import traceback
        print(traceback.format_exc())
        pass

    return items


def get_confirm_notifs(client, res):
    b = bs4.BeautifulSoup(res['domops'][0][3]['__html'], 'html.parser')
    lis = b.find_all('li', class_='friendConfirmedNotifsUnitAggregated')
    maxtime = 120

    def _get_li_info(li):
        abbr = li.find('abbr')
        if abbr and abbr.has_attr('data-utime'):
            timestamp = int(abbr.get('data-utime'))
            now = int(time.time())
            if now - timestamp > maxtime:
                return None

        span_ = li.find('span')  # name
        main_page = li.find('a').get('href')
        return (main_page, span_.text)

    all_friend_main = []

    for li in lis:
        a_label = li.find('a')
        if a_label:
            abbr = a_label.find('abbr')
            if abbr and abbr.has_attr('data-utime'):
                timestamp = int(abbr.get('data-utime'))
                now = int(time.time())
                if now - timestamp > maxtime:
                    break
            if a_label.has_attr('ajaxify'):  # 合并详情
                url = a_label.get('ajaxify')
                data = dict(furl.furl(url).args)
                res = client._post(url, data)  # fbProfileBrowserListItem
                html = res['jsmods']['markup'][0][1]['__html']
                b = bs4.BeautifulSoup(html, 'html.parser')
                new_lis = b.find_all('li', 'fbProfileBrowserListItem')
                for x in new_lis:
                    all_friend_main.append(_get_li_info(x))

            elif a_label.has_attr('href'):
                all_friend_main.append(_get_li_info(li))

    return [x for x in all_friend_main if x]


def hovercard_get_addfriend(res):
    b = bs4.BeautifulSoup(res['jsmods']['markup'][0][1]['__html'], 'html.parser')
    button = b.find('button', class_='FriendRequestAdd')
    return not button is None


def get_ext_data(res):
    if isinstance(res, dict):
        try:
            for x in res.get('jsmods').get('require'):
                if len(x) == 4 and x[1] == 'pageletComplete':
                    return x[3][0]
        except:
            return {}
    else:
        try:
            global_data = re.findall(r'globalData:(\{.*?display_params.*?\}),prefetchPixels', res.text)[0]
            data = demjson.decode(global_data)
            complete = re.findall('"pageletComplete",\[\],\[(.*?)\]', res.text)[0]
            data.update(demjson.decode(complete))
            return data
        except:
            return {}


def get_all_group(text):
    items = []
    if 'BrowseResultsContainer' in text:
        b = bs4.BeautifulSoup(text, 'html.parser')
        pattern = re.compile(r'BrowseResultsContainer')
        elem = b.find_all('div', class_='hidden_elem', string=pattern)
        for e in elem:
            div = bs4.BeautifulSoup(e.string, 'html.parser')
            iter = div.find('div', id='BrowseResultsContainer').childGenerator()
            for item in iter:
                try:
                    data_bt = item.get('data-bt')
                    raw_id = json.loads(data_bt).get('id')
                    a_label = item.find('a')
                    name = a_label.get('aria-label')
                    items.append({'id': raw_id, 'name': name})
                except Exception as e:
                    print(e)
                    pass
    elif 'data-testid' in text:
        b = bs4.BeautifulSoup(text, 'html.parser')
        divs = b.find_all(lambda tag: tag.name == 'div' and tag.has_attr('data-bt'))
        for item in divs:
            try:
                data_bt = item.get('data-bt')
                if 'query_data' not in data_bt:
                    continue
                raw_id = json.loads(data_bt).get('id')
                a_label = item.find('a')
                name = a_label.get('aria-label')
                items.append({'id': raw_id, 'name': name})
            except Exception as e:
                print(e)
                pass
    return items


def get_members_ids(text):
    def get_member(text):
        valid = []
        div = bs4.BeautifulSoup(text, 'html.parser')
        try:
            member_num = div.find(
                lambda tag: tag.name == 'a' and "/members/" in tag.attrs.get('href', '')).find_next_sibling('span').text
            assert member_num
            member_num = int(member_num.replace(',', ''))
        except:
            member_num = 0
        more_page_a = div.find(lambda tag: tag.name == 'a' and 'group_confirmed_members' in tag.attrs.get('href', ''))
        if not more_page_a:
            return [], '', member_num
        more_url = more_page_a.attrs.get('href')
        btn_divs = div.find_all('div', class_='FriendButton')
        for div in btn_divs:
            btn = div.find(lambda tag: tag.name == 'button' and tag.has_attr('data-profileid'))
            valid.append(btn.get('data-profileid'))
        return valid, more_url, member_num

    b = bs4.BeautifulSoup(text, 'html.parser')
    if 'groupsMemberBrowser' in text:
        pattern = re.compile(r'groupsMemberBrowser')
        elem = b.find('div', class_='hidden_elem', string=pattern)
        return get_member(elem.string)
    else:
        return get_member(text)


def checkpoint_text(text):
    try:
        b = bs4.BeautifulSoup(text, 'html.parser')
        return b.find('form', class_='checkpoint').find('span').text
    except:
        return ""


def get_location_info(html):
    try:
        selector = etree.HTML(html)
        return selector.xpath('//div[@id="content"]/div/div/div[1]/div/div/div[2]/div/strong')[0].text
    except:
        return "未知位置"


def get_photo_attachments_list(text):
    try:
        b = bs4.BeautifulSoup(text, 'html.parser')
        scr = b.find('script', string=re.compile(r'photo_attachments_list')).text
        all_ = re.findall(r'photo_attachments_list\.\[(.*?)\]', scr)[0]
        return all_.split(',')
    except:
        return []


def parse_places_group(text):
    try:
        b = bs4.BeautifulSoup(text, 'html.parser')
        search_script = b.find('script', string=re.compile('PlacesSearchResults'))
        results_query = re.compile(r'{data:{results:{(.*?)},query:"')
        result = demjson.decode('{' + results_query.findall(search_script.string)[0] + '}')
        cursor_match = re.compile(r'cursor:"(.*?)"')
        cursor = cursor_match.findall(search_script.string)[0]
        return result, cursor
    except:
        return [], None


def get_fb_dtsg(c):
    try:
        return [x[2]['token'] for x in c['jsmods']['define'] if x[0] == 'DTSGInitialData'][0]
    except:
        return None


def get_reaction_profile(res):
    html = get_domops_3(res)
    b = bs4.BeautifulSoup(html, 'html.parser')
    res_list = []
    for li in b.contents:
        data = li.find('a').attrs
        if 'data-hovercard' not in data:
            continue
        name = data.get('title')
        id = re.findall(r'user.php\?id=(\d+)&', data['data-hovercard'])[0]
        res_list.append({'id': id, 'name': name})
    return res_list


def get_approval_form(content):
    b = bs4.BeautifulSoup(content, 'html.parser')
    pattern = re.compile(r'fbTimelineLogColumn')
    elem = b.find('div', class_='hidden_elem', string=pattern)
    b = bs4.BeautifulSoup(elem.string, 'html.parser')
    form = b.find('form', attrs={'action': re.compile("approval/respond.php")})
    url = form.attrs.get('ajaxify')
    params = {input_.attrs['name']: input_.attrs['value'] for input_ in form.find_all('input') if
              input_.attrs.get('value')}
    params['approve_button'] = '1'
    return url, params