-
Notifications
You must be signed in to change notification settings - Fork 9
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[WIP] make tmsc work with git-based modelforge and sourced.ml #12
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
sourced-ml==0.5.1 | ||
ast2vec>=0.3.8-alpha |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,14 +2,16 @@ | |
import json | ||
import logging | ||
import sys | ||
import os | ||
|
||
from ast2vec import Topics, DocumentFrequencies, DEFAULT_BBLFSH_TIMEOUT | ||
from ast2vec.bow import BOWBase | ||
from sourced.ml.models import BOW, Topics, DocumentFrequencies | ||
from modelforge.backends import create_backend | ||
from modelforge.index import GitIndex | ||
|
||
from tmsc.environment import initialize | ||
from tmsc.topic_detector import TopicDetector | ||
|
||
DEFAULT_BBLFSH_TIMEOUT = 20 | ||
|
||
def main(): | ||
parser = argparse.ArgumentParser() | ||
|
@@ -33,6 +35,10 @@ def main(): | |
parser.add_argument("--prune-df", default=20, type=int, | ||
help="Minimum number of times an identifer must occur in different " | ||
"documents to be taken into account.") | ||
parser.add_argument("--index_repo", default="https://github.com/src-d/models", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These two are normally hardcoded in I think that args.topics = Topics(log_level=args.log_level).load(source=args.topics) will work - the backend will be created automatically. If it doesn't then there is a bug somewhere. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. seems like a 🐛 confirmed import logging
from sourced.ml.models import Topics
topics = Topics(log_level=logging.INFO).load(source=None) works as expected only if there is warm cache already in In case if the cache is empty, the very same code results in
|
||
help="Models index repository.") | ||
parser.add_argument("--index_cache", default=os.path.join(BOW.cache_dir(), "models"), | ||
help="Local cache of models index repository") | ||
parser.add_argument("-n", "--nnn", default=10, type=int, | ||
help="Number of topics to print.") | ||
parser.add_argument("-f", "--format", default="human", choices=["json", "human"], | ||
|
@@ -42,24 +48,26 @@ def main(): | |
if args.linguist is None: | ||
args.linguist = "./enry" | ||
initialize(args.log_level, enry=args.linguist) | ||
|
||
if args.gcs: | ||
backend = create_backend(args="bucket=" + args.gcs) | ||
else: | ||
backend = create_backend() | ||
if args.topics is not None: | ||
args.topics = Topics(log_level=args.log_level).load(source=args.topics, backend=backend) | ||
if args.df is not None: | ||
args.df = DocumentFrequencies(log_level=args.log_level).load( | ||
source=args.df, backend=backend) | ||
if args.bow is not None: | ||
args.bow = BOWBase(log_level=args.log_level).load(source=args.bow, backend=backend) | ||
git_index = GitIndex(index_repo=args.index_repo, cache=args.index_cache, log_level=args.log_level) | ||
backend = create_backend(git_index=git_index) | ||
|
||
args.topics = Topics(log_level=args.log_level).load(source=args.topics, backend=backend) #source=args.topics | ||
args.df = DocumentFrequencies(log_level=args.log_level).load(source=args.df, backend=backend) | ||
#args.bow = BOW(log_level=args.log_level).load(source=args.bow, backend=backend) | ||
|
||
sr = TopicDetector( | ||
topics=args.topics, docfreq=args.df, bow=args.bow, verbosity=args.log_level, | ||
prune_df_threshold=args.prune_df, gcs_bucket=args.gcs, repo2bow_kwargs={ | ||
prune_df_threshold=args.prune_df, repo2bow_kwargs={ | ||
"linguist": args.linguist, | ||
"bblfsh_endpoint": args.bblfsh, | ||
"timeout": args.timeout}) | ||
|
||
topics = sr.query(args.input, size=args.nnn) | ||
|
||
if args.format == "json": | ||
json.dump({"repository": args.input, "topics": topics}, sys.stdout) | ||
elif args.format == "human": | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
from collections import defaultdict | ||
import marshal | ||
import math | ||
import types | ||
|
||
from sourced.ml.models import BOW, DocumentFrequencies | ||
from sourced.ml.algorithms.uast_ids_to_bag import UastIds2Bag | ||
|
||
class Uasts2BOW: | ||
def __init__(self, vocabulary: dict, docfreq: DocumentFrequencies, | ||
getter: callable): | ||
self._docfreq = docfreq | ||
self._uast2bag = UastIds2Bag(vocabulary) #TODO replace with sourced.ml. | ||
self._reverse_vocabulary = [None] * len(vocabulary) | ||
for key, val in vocabulary.items(): | ||
self._reverse_vocabulary[val] = key | ||
self._getter = getter | ||
|
||
@property | ||
def vocabulary(self): | ||
return self._uast2bag.token2index #.vocabulary | ||
|
||
@property | ||
def docfreq(self): | ||
return self._docfreq | ||
|
||
def __call__(self, file_uast_generator): | ||
freqs = defaultdict(int) | ||
for file_uast in file_uast_generator: | ||
bag = self._uast2bag(self._getter(file_uast)) #.uast_to_bag | ||
for key, freq in bag.items(): | ||
freqs[key] += freq | ||
missing = [] | ||
for key, val in freqs.items(): | ||
try: | ||
freqs[key] = math.log(1 + val) * math.log( | ||
self._docfreq.docs / self._docfreq[self._reverse_vocabulary[key]]) | ||
except KeyError: | ||
missing.append(key) | ||
for key in missing: | ||
del freqs[key] | ||
return dict(freqs) | ||
|
||
def __getstate__(self): | ||
state = self.__dict__.copy() | ||
if isinstance(self._getter, types.FunctionType) \ | ||
and self._getter.__name__ == (lambda: None).__name__: | ||
assert self._getter.__closure__ is None | ||
state["_getter"] = marshal.dumps(self._getter.__code__) | ||
return state | ||
|
||
def __setstate__(self, state): | ||
self.__dict__ = state | ||
if isinstance(self._getter, bytes): | ||
self._getter = types.FunctionType(marshal.loads(self._getter), globals()) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ast2vec is dead
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I know, was kept just as a hack to satisfy internal imports 😕