Asynchronous retrieval of source code and dependencies

[rilma]

import re
from typing import List
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen

def _get_links(url: str, pattern: str=r'.') -> List[str]:
    html_page = urlopen(Request(url))
    soup = BeautifulSoup(html_page, "html.parser")
    links = [link.get('href') for link in soup.findAll('a') if re.match(pattern, link.get('href'), re.DOTALL)]
    return links

FILENAME_REGEX_PATTERN = r'[a-z0-9_]+.[a-z]+'

import os; os.system(f'ls -lah {self.temporary_directory.name}')