Merge branch 'FIX/LinkedinAuth' into 'iKy'

Linkedin Authentication and more See merge request kennbroorg/iKy!117
kennbroorg · May 3, 2021 · 415070a · 415070a
2 parents 63c2b91 + a59c98c
commit 415070a
Show file tree

Hide file tree

Showing 14 changed files with 2,591 additions and 2,713 deletions.
diff --git a/README.md b/README.md
@@ -216,7 +216,7 @@ Once the application is loaded in the browser, you should go to the Api Keys opt
 
 - Fullcontact: Generate the APIs from [here](https://support.fullcontact.com/hc/en-us/articles/115003415888-Getting-Started-FullContact-v2-APIs)
 - PeopleDataLabs : Generate the APIs from [aquí](https://www.peopledatalabs.com/signup)
-- Linkedin: Only the user and password of your account must be loaded
+- **Linkedin** : Linkedin changed their authentication process and with that the module in iKy stopped working, but I found a relatively easy way to baypass it All you have to do is log into linkedin in a Firefox browser. Yes, just that iKy will look for the cookie, use it and scrape the information. I hope it works for a long time
 - Instagram: Only the user and password of your account must be loaded
 - HaveIBeenPwned : Generate the APIs from [here](https://haveibeenpwned.com/API/Key) (Paid)
 - Emailrep.io : Generate the APIs from [here](https://emailrep.io/key)
@@ -231,14 +231,16 @@ Once the application is loaded in the browser, you should go to the Api Keys opt
 - [iKy Page](https://kennbroorg.gitlab.io/ikyweb/)
 - Installation
   - [Easy Install](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/EasyInstall)
-  - [Vagrant](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/Vagrant)
   - [Manual install (Compacted)](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/Manual-install-(Compacted))
   - [Manual install (Detailed)](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/Manual-install-(Detailed))
+  - [Docker](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/Docker)
+  - [Vagrant](https://gitlab.com/kennbroorg/iKy/-/wikis/Installation/Vagrant)
 - Update
   - [Soft Update](https://gitlab.com/kennbroorg/iKy/-/wikis/Update/Soft)
 - Wake Up 
   - [Turn on the project](https://gitlab.com/kennbroorg/iKy/-/wikis/Wakeup/WakeUp)
 - APIs
+  - [Get APIs](https://gitlab.com/kennbroorg/iKy/-/wikis/APIs/ApiKeys-get)
   - [APIs through frontend](https://gitlab.com/kennbroorg/iKy/-/wikis/APIs/ApiKeys-through-the-browser)
   - [APIs through backend](https://gitlab.com/kennbroorg/iKy/-/wikis/APIs/APIs-through-the-backend)
 - Backend
@@ -260,7 +262,7 @@ Once the application is loaded in the browser, you should go to the Api Keys opt
 ## Demo Video
 
 <div align="center">
-    <a href="https://vimeo.com/434501702"><img alt="Kali 2019" src="https://kennbroorg.gitlab.io/ikyweb/assets/img/iKy-01.png"></a>
+    <a href="https://vimeo.com/496879025"><img alt="iKy SOL" src="https://kennbroorg.gitlab.io/ikyweb/assets/img/iKy-01.png"></a>
     <p>Vimeo</p>
 </div>
 

diff --git a/backend/modules/holehe/holehe_tasks.py b/backend/modules/holehe/holehe_tasks.py
@@ -29,14 +29,18 @@
 
 async def p_holehe(email, from_m):
 
+    holehe.core.check_if_email(email)
+
     modules = holehe.core.import_submodules("holehe.modules")
     websites = holehe.core.get_functions(modules)
 
     client = httpx.AsyncClient(timeout=60)
     out = []
     async with trio.open_nursery() as nursery:
         for website in websites:
-            nursery.start_soon(website, email, client, out)
+            nursery.start_soon(holehe.core.launch_module, website, email, 
+                               client, out)
+            # nursery.start_soon(website, email, client, out)
     raw_node = sorted(out, key=lambda i: i['name'])  # We sort by modules names
     await client.aclose()
 

diff --git a/backend/modules/linkedin/linkedin_tasks.py b/backend/modules/linkedin/linkedin_tasks.py
@@ -4,8 +4,10 @@
 import sys
 import json
 import requests
-import os
+import time
+import traceback
 from collections import Counter
+import browser_cookie3
 
 import re
 from bs4 import BeautifulSoup
@@ -32,13 +34,6 @@
 
 logger = get_task_logger(__name__)
 
-# Compatibility code
-try:
-    # Python 2: "unicode" is built-in
-    unicode
-except NameError:
-    unicode = str
-
 
 def date_convert(date):
     date_conv = str(date.get("year", ""))
@@ -47,101 +42,116 @@ def date_convert(date):
     return date_conv
 
 
-@celery.task
-def t_linkedin(email, from_m):
-
-    cookie_file = os.path.join(os.path.dirname(
-        os.path.realpath(__file__)), "cookie-linkedin.json")
+def p_linkedin(user, from_m):
 
-    # cookie_file = "cookie-linkedin.json"
-    expiration_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
+    # import pdb; pdb.set_trace()
 
     s = requests.Session()
-    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A'}
-
-    if (os.path.isfile(cookie_file)
-        and (expiration_date < datetime.fromtimestamp(
-            os.path.getmtime(cookie_file)).strftime('%Y-%m-%d'))):
-
-        with open(cookie_file, 'r') as f:
-            s.cookies = requests.utils.cookiejar_from_dict(json.load(f))
-
+    headers = {'User-Agent': 
+               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) ' + 
+               'AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 ' +
+               'Safari/7046A194A'}
+
+    # Try to get cookie from browser
+    # TODO: Write valid cookies in apikeys
+    ref = ["chromium", "opera", "edge", "firefox", "chrome"]
+    index = 0
+    json_cookie = {}
+    found = False
+    for cookie_fn in [
+        browser_cookie3.chromium,
+        browser_cookie3.opera,
+        browser_cookie3.edge,
+        browser_cookie3.firefox,
+        browser_cookie3.chrome,
+    ]:
+        try:
+            for cookie in cookie_fn(domain_name=""):
+                if ('linkedin' in cookie.domain):
+                    if ((cookie.name == 'li_at' or 
+                         cookie.name == 'JSESSIONID') and 
+                            (not cookie.is_expired())):
+                        json_cookie['browser'] = ref[index]
+                        json_cookie[cookie.name] = cookie.value
+                        json_cookie[cookie.name + '_expires'] = cookie.expires
+
+                # Check
+                if((json_cookie.get("li_at", "") != "") and
+                   (json_cookie.get("JSESSIONID", "") != "")):
+                    found = True
+                    break
+        except Exception as e:
+            print(e)
+
+        index += 1
+
+        if(found):
+            break
+
+    if(found):
+        s.cookies['li_at'] = json_cookie['li_at']
+        s.cookies['JSESSIONID'] = json_cookie['JSESSIONID']
+        s.headers = headers
+        s.headers["csrf-token"] = s.cookies["JSESSIONID"].strip('"')
     else:
-        # Get user and pass
-        linkedin_user = api_keys_search('linkedin_user')
-        linkedin_pass = api_keys_search('linkedin_pass')
+        raise RuntimeError('iKy can\'t detect cookies')
 
-        # Login process
-        r = s.get('https://www.linkedin.com/login?trk=guest_homepage-basic_nav-header-signin', headers=headers)
-        cookies = dict(r.cookies)
+    url = 'https://www.linkedin.com/voyager/api/identity/profiles/' + \
+        user + '/profileView'
 
-        soup = BeautifulSoup(r.content, "lxml")
-        hidden_tags = soup.find_all("input", type="hidden")
-        payload = {}
-        for input_hidden in hidden_tags:
-            payload[input_hidden['name']] = input_hidden['value']
-        payload['session_key'] = linkedin_user
-        payload['session_password'] = linkedin_pass
+    req = s.get(url)
 
-        req = s.post("https://www.linkedin.com/checkpoint/lg/login-submit",
-                     data=payload,
-                     headers=headers)
-
-        with open('cookie-linkedin.json', 'w') as f:
-            json.dump(requests.utils.dict_from_cookiejar(s.cookies), f)
+    # Get ID
+    ids = []
+    match = re.search(r'\"profileId\":\"(\w*)\"', req.text)
+    if (match):
+        ids = match.groups()[0].strip()
+    else:
+        raise RuntimeError('Linkedin user don\'t exist')
 
-    for cookie in s.cookies:
-        if "JSESSIONID" in str(cookie):
-            csrfToken = re.findall('JSESSIONID=([^ ]*)', str(cookie))
 
     # Total
     total = []
     total.append({'module': 'linkedin'})
-    total.append({'param': email})
+    total.append({'param': user})
 
     found = False
     # Get profile from username
-    if ("http" in email):
+    if ("http" in user):
         total.append({'validation': 'hard'})
-        url = email
+        url = user
         found = True
+        raise RuntimeError('Linkedin don\'t work with url')
     # Get profile from email
-    elif ("@" in email):
-        total.append({'validation': 'hard'})
-        url = "https://www.linkedin.com/sales/gmail/profile/" + \
-            "viewByEmail/%s" % email
-        req = s.get(url, headers=headers)
-        if "Sorry, we couldn't find a matching LinkedIn" not in req.text:
-            soup = BeautifulSoup(req.content, "lxml")
-            a = soup.find("a", {"id": "profile-link"})
-            url = a['href']
-            found = True
+    elif ("@" in user):
+        raise RuntimeError('Linkedin don\'t work with email')
     else:
         total.append({'validation': 'hard'})
-        url = "https://www.linkedin.com/in/%s/" % email
+        url = "https://www.linkedin.com/in/%s/" % user
         found = True
     # Get profile from http
 
     if found:
+        s.headers["csrf-token"] = s.cookies["JSESSIONID"].strip('"')
+        headers["csrf-token"] = s.cookies["JSESSIONID"].strip('"')
+
         req = s.get(url, headers=headers)
 
-        ids = re.findall(
-            '\/voyager\/api\/identity\/profiles\/(.*)\/',
-            req.text)
-        id = Counter(ids).most_common(1)
+        id = Counter([ids]).most_common(1)  # This is for compatibility code
 
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + "/following?q=followedEntities&count=17"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+
         req = s.get(url, headers=headers)
+
         # Following
         following_view = json.loads(req.text)
 
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + "/skillCategory?includeHiddenEndorsers=true"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+        # s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
         req = s.get(url, headers=headers)
-        content = re.sub("\. ?\n", ",\n", unicode(req.text))
+        content = re.sub("\. ?\n", ",\n", req.text)
         content = re.sub(" = ", " : ", content)
         # Skill Category
         skill_category = json.loads(content)
@@ -157,24 +167,24 @@ def t_linkedin(email, from_m):
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + \
             "/recommendations?q=received&recommendationStatuses=List(VISIBLE)"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+        # s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
         req = s.get(url, headers=headers)
         # Recomendaciones 1
         recommend_received = json.loads(req.text)
 
         # TODO : KKK : For person relationship
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + "/recommendations?q=given"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+        # s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
         req = s.get(url, headers=headers)
         # Recomendaciones 2
         recommend_given = json.loads(req.text)
 
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + "/profileView"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+        # s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
         req = s.get(url, headers=headers)
-        content = re.sub("\. ?\n", ",\n", unicode(req.text))
+        content = re.sub("\. ?\n", ",\n", req.text)
         content = re.sub(" = ", " : ", content)
         # Profile
         profile_view = json.loads(content)
@@ -188,9 +198,9 @@ def t_linkedin(email, from_m):
 
         url = "https://www.linkedin.com/voyager/api/identity/profiles/" + \
             id[0][0] + "/networkinfo"
-        s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
+        # s.headers.update({'csrf-token': csrfToken[0].replace('"', '')})
         req = s.get(url, headers=headers)
-        content = re.sub("\. ?\n", ",\n", unicode(req.text))
+        content = re.sub("\. ?\n", ",\n", req.text)
         content = re.sub(" = ", " : ", content)
         # NetworkInfo
         network_info = json.loads(content)
@@ -223,19 +233,12 @@ def t_linkedin(email, from_m):
         # PrivacySettings
         # x = json.loads(req.text)
 
-        raw = []
-        raw.append({'profile_view': profile_view})
-        raw.append({'network_info': network_info})
-        raw.append({'recommend_received': recommend_received})
-        raw.append({'recommend_given': recommend_given})
-        raw.append({'following_view': following_view})
-        raw.append({'skill_category': skill_category})
-
         # Graphic Array
         graphic = []
 
         # Profile Array
         profile = []
+        presence = []
 
         # Timeline Array
         timeline = []
@@ -487,6 +490,16 @@ def t_linkedin(email, from_m):
                                "value": s.get("endorsementCount", "")})
         skill_tmp = {"name": "", "value": 100, "children": skills}
 
+        # Raw with status
+        raw = []
+        raw.append({"code": 0})
+        raw.append({'profile_view': profile_view})
+        raw.append({'network_info': network_info})
+        raw.append({'recommend_received': recommend_received})
+        raw.append({'recommend_given': recommend_given})
+        raw.append({'following_view': following_view})
+        raw.append({'skill_category': skill_category})
+
         total.append({'raw': raw})
         graphic.append({'social': socialp})
         # graphic.append({'skills': skills})
@@ -502,12 +515,61 @@ def t_linkedin(email, from_m):
         # graphic.append({'educationView': educationView})
         total.append({'graphic': graphic})
 
+        presence.append({"name": "Linkedin",
+                         "children": [
+                             {"name": "followers", 
+                              "value": network_info.get("followersCount", "")},
+                             {"name": "following", 
+                              "value": following_view.get("paging", "").get(
+                                                          "total", "")}
+                         ]})
+
+        profile.append({'presence': presence})
+
         total.append({'profile': profile})
         total.append({'timeline': timeline})
 
     return total
 
 
+@celery.task
+def t_linkedin(user, from_m):
+    # Variable principal
+    total = []
+    # Take initial time
+    tic = time.perf_counter()
+
+    # try execution principal function
+    try:
+        total = p_linkedin(user, from_m)
+    # Error handle
+    except Exception as e:
+        # Error description
+        traceback.print_exc()
+        traceback_text = traceback.format_exc()
+
+        # Set module name in JSON format
+        total.append({"module": "linkedin"})
+
+        # Set status code and reason
+        status = []
+        status.append(
+            {
+                "code": 10,  # this code is arbitrary
+                "reason": "{}".format(e),
+                "traceback": traceback_text,
+            }
+        )
+        total.append({"raw": status})
+
+    # Take final time
+    toc = time.perf_counter()
+    # Show process time
+    logger.info(f"Linkedin - Response in {toc - tic:0.4f} seconds")
+
+    return total
+
+
 def output(data):
     print(json.dumps(data, ensure_ascii=True, indent=2))