diff --git a/.github/workflows/cs102.yml b/.github/workflows/cs102.yml index 4509ac4..2df86e7 100644 --- a/.github/workflows/cs102.yml +++ b/.github/workflows/cs102.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8.6 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: '3.8.6' + python-version: '3.9' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/Waycoolers.iml b/.idea/Waycoolers.iml new file mode 100644 index 0000000..4c883db --- /dev/null +++ b/.idea/Waycoolers.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..1260ee9 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..67df6a8 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/homework05/research/age.py b/homework05/research/age.py index 492ae28..ecd3edc 100644 --- a/homework05/research/age.py +++ b/homework05/research/age.py @@ -14,4 +14,23 @@ def age_predict(user_id: int) -> tp.Optional[float]: :param user_id: Идентификатор пользователя. :return: Медианный возраст пользователя. """ - pass + list_friends = get_friends(user_id=user_id, fields=["bdate"]) + ret = [] + year = dt.datetime.now().year + month = dt.datetime.now().month + day = dt.datetime.now().day + for i in list_friends.items: + if "bdate" in i: # type: ignore + date = list(map(int, i["bdate"].split("."))) # type: ignore + if len(date) != 3: + continue + age = ( + -date[2] + + year + - (1 if (date[1] < month or (date[1] == month and day < date[0])) else 0) + ) + ret.append(age) + if not len(ret): + return None + + return statistics.median(ret) diff --git a/homework05/research/network.py b/homework05/research/network.py index 6b6db7c..8e46ab9 100644 --- a/homework05/research/network.py +++ b/homework05/research/network.py @@ -5,7 +5,6 @@ import matplotlib.pyplot as plt import networkx as nx import pandas as pd - from vkapi.friends import get_friends, get_mutual @@ -18,7 +17,13 @@ def ego_network( :param user_id: Идентификатор пользователя, для которого строится граф друзей. :param friends: Идентификаторы друзей, между которыми устанавливаются связи. """ - pass + ret = [] + mutual_friends = get_mutual(source_uid=user_id, target_uids=friends) + for person in mutual_friends: + if person is not None: + for j in person["common_friends"]: # type: ignore + ret.append((person["id"], j)) # type: ignore + return ret def plot_ego_network(net: tp.List[tp.Tuple[int, int]]) -> None: diff --git a/homework05/research/topic_modeling.py b/homework05/research/topic_modeling.py index 7be95e5..d46c011 100644 --- a/homework05/research/topic_modeling.py +++ b/homework05/research/topic_modeling.py @@ -3,7 +3,6 @@ from gensim.corpora import Dictionary from textacy import preprocessing from tqdm import tqdm - from vkapi.wall import get_wall_execute diff --git a/homework05/tests/tests_api/test_friends.py b/homework05/tests/tests_api/test_friends.py index d5493aa..96133f2 100644 --- a/homework05/tests/tests_api/test_friends.py +++ b/homework05/tests/tests_api/test_friends.py @@ -3,7 +3,6 @@ import unittest import responses - from vkapi.friends import FriendsResponse, get_friends, get_mutual diff --git a/homework05/tests/tests_api/test_session.py b/homework05/tests/tests_api/test_session.py index 895028b..c399906 100644 --- a/homework05/tests/tests_api/test_session.py +++ b/homework05/tests/tests_api/test_session.py @@ -3,8 +3,12 @@ import httpretty import responses -from requests.exceptions import ConnectionError, HTTPError, ReadTimeout, RetryError - +from requests.exceptions import ( # mypy: ignore-errors + ConnectionError, + HTTPError, + ReadTimeout, + RetryError, +) from vkapi.session import Session diff --git a/homework05/tests/tests_api/test_wall.py b/homework05/tests/tests_api/test_wall.py index 6a56a8d..8ab2c26 100644 --- a/homework05/tests/tests_api/test_wall.py +++ b/homework05/tests/tests_api/test_wall.py @@ -5,7 +5,6 @@ import pandas as pd import responses - from vkapi.wall import get_wall_execute diff --git a/homework05/tests/tests_research/test_age.py b/homework05/tests/tests_research/test_age.py index 404fd0a..53b941c 100644 --- a/homework05/tests/tests_research/test_age.py +++ b/homework05/tests/tests_research/test_age.py @@ -2,7 +2,6 @@ import unittest import responses - from research.age import age_predict diff --git a/homework05/tests/tests_research/test_network.py b/homework05/tests/tests_research/test_network.py index 3608565..e375473 100644 --- a/homework05/tests/tests_research/test_network.py +++ b/homework05/tests/tests_research/test_network.py @@ -2,7 +2,6 @@ import unittest import responses - from research.network import ego_network diff --git a/homework05/vkapi/config.py b/homework05/vkapi/config.py index 8459497..313c62a 100644 --- a/homework05/vkapi/config.py +++ b/homework05/vkapi/config.py @@ -2,6 +2,6 @@ VK_CONFIG = { "domain": "https://api.vk.com/method", - "access_token": "", + "access_token": "https://oauth.vk.com/blank.html#access_token=vk1.a.MQmRX7cHcqF_La-YErIO8ZhRfpsltLJS2bOj0TYaYgULRj6a79mRo685eQUe0hTZNrnZYuufUHvCqsg09jTvbXSYddaWawXpf5RjtFi0GU7lPCTmfOeqlo4xuqZfb-w-hX4zOfulVhDzOqB4ttzSNBY1aM5FPSdA-AT-EY3_7ykzjhPIfUtZYAQ5VcnysL5q&expires_in=86400&user_id=203091488", "version": "5.126", } diff --git a/homework05/vkapi/friends.py b/homework05/vkapi/friends.py index dad6a5c..826b4b7 100644 --- a/homework05/vkapi/friends.py +++ b/homework05/vkapi/friends.py @@ -28,7 +28,16 @@ def get_friends( :param fields: Список полей, которые нужно получить для каждого пользователя. :return: Список идентификаторов друзей пользователя или список пользователей. """ - pass + list_of_friends = session.get( + "friends.get", + user_id=user_id, + count=count, + offset=offset, + fields=fields, + version=config.VK_CONFIG["version"], + access_token=config.VK_CONFIG, + ).json()["response"] + return FriendsResponse(count=list_of_friends["count"], items=list_of_friends["items"]) class MutualFriends(tp.TypedDict): @@ -57,4 +66,24 @@ def get_mutual( :param offset: Смещение, необходимое для выборки определенного подмножества общих друзей. :param progress: Callback для отображения прогресса. """ - pass + ret = [] + ln = 1 + if target_uids is not None: + ln = (int(len(target_uids)) + 99) // 100 + + for i in range(0, ln): + ret += session.get( + "friends.getMutual", + source_uid=source_uid, + target_uid=target_uid, + target_uids=target_uids, + order=order, + count=count, + offset=i * 100, + version=config.VK_CONFIG["version"], + access_token=config.VK_CONFIG["access_token"], + ).json()["response"] + if i % 3 == 2: + time.sleep(1) + + return ret diff --git a/homework05/vkapi/session.py b/homework05/vkapi/session.py index 0643389..e581a02 100644 --- a/homework05/vkapi/session.py +++ b/homework05/vkapi/session.py @@ -17,15 +17,35 @@ class Session: def __init__( self, - base_url: str, + base_url: str = "https://", timeout: float = 5.0, max_retries: int = 3, backoff_factor: float = 0.3, ) -> None: - pass + self.current_session = requests.session() + retry_strategy = Retry( + total=max_retries, + backoff_factor=backoff_factor, + status_forcelist=[429, 500, 502, 503, 504], + method_whitelist=["HEAD", "GET", "OPTIONS"], + ) + adapter = HTTPAdapter( + max_retries=retry_strategy + ) # повторяет попытку соединения в случае неудачи + self.current_session.mount( + base_url, adapter=adapter + ) # регистрируем экземпляр адаптера в префиксе + self.base_url = base_url + self.timeout = timeout - def get(self, url: str, *args: tp.Any, **kwargs: tp.Any) -> requests.Response: - pass + def get( + self, url: str, *args: tp.Any, **kwargs: tp.Any + ) -> requests.Response: # запрос содержимого с определенного ресурса + return self.current_session.get( + self.base_url + "/" + url, params=kwargs, timeout=self.timeout + ) - def post(self, url: str, *args: tp.Any, **kwargs: tp.Any) -> requests.Response: - pass + def post( + self, url: str, *args: tp.Any, **kwargs: tp.Any + ) -> requests.Response: # передача пользовательских данных заданному ресурсу + return self.current_session.post(self.base_url + "/" + url, data=kwargs) diff --git a/homework05/vkapi/wall.py b/homework05/vkapi/wall.py index a045c6c..8e39969 100644 --- a/homework05/vkapi/wall.py +++ b/homework05/vkapi/wall.py @@ -5,7 +5,6 @@ import pandas as pd from pandas import json_normalize - from vkapi import config, session from vkapi.exceptions import APIError @@ -20,7 +19,7 @@ def get_posts_2500( extended: int = 0, fields: tp.Optional[tp.List[str]] = None, ) -> tp.Dict[str, tp.Any]: - pass + return {} def get_wall_execute( @@ -49,4 +48,42 @@ def get_wall_execute( :param fields: Список дополнительных полей для профилей и сообществ, которые необходимо вернуть. :param progress: Callback для отображения прогресса. """ - pass + code = f""" + if({count} < 100): + retVal = API.wall.get( + {{ + "owner_id":{owner_id}, + "domain":{domain}, + "offset":{offset}, + "count":"{count}", + "filter":{filter}, + "extended":{extended}, + "fields": {fields} + "v":{config.VK_CONFIG["version"]} + }} + ) + else : + retVal =[] + for i in range(0, Math.floor({count} / 100) - 1, 100): + answer=API.wall.get({{ + "owner_id": str = {owner_id}, + "domain": str = {domain}, + "offset": int = {offset} + i * 100, + "count": int = 100, + "filter": str = {filter}, + "extended": str = {extended}, + "fields": str = {fields}, + "v":{config.VK_CONFIG["version"]} + }}) + retVal.push(...answer) + return retVal + """ + time.sleep(2) + response = session.post( + "execute", + code=code, + access_token=config.VK_CONFIG["access_token"], + version=config.VK_CONFIG["version"], + ).json()["response"]["items"] + + return json_normalize(response) diff --git a/requirements.txt b/requirements.txt index 868f6a2..e0e2794 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,17 @@ isort==5.4.2 mypy==0.782 pylint==2.6.0 pytest==6.0.1 + +setuptools~=49.2.1 +pandas~=1.5.3 +responses~=0.22.0 +vkapi~=1.1 +httpretty~=1.1.4 +requests~=2.28.2 +community~=1.0.0b1 +matplotlib~=3.7.0 +networkx~=3.0 +gensim~=4.3.0 +pyLDAvis~=3.4.0 +textacy~=0.12.0 +tqdm~=4.64.1 \ No newline at end of file