Skip to content

Commit

Permalink
Update on "[jit] Polymorphic IValue::type() for DynamicType."
Browse files Browse the repository at this point in the history
Before the change:
```
c10::Type t = ivalue.type();
```
After the change:
```
c10::Type t = ivalue.type();
c10::DynamicType d = ivalue.type<c10::DynamicType>(); // new path
```
The new path will be adopted in PyTorch Lite Interpreter to support lightweight type reflection. Note that type getters are selected at compile time so no performance overhead. The benefits of having a DynamicType will be elaborated in a separate document, but in short, DynamicType provides an isolated type system for controlling binary size bloat, and shrink down ~20 supported Type symbols into one so that the size taken by specializations and function name symbols are greatly reduced.

Lite Interpreter should only use the `<DynamicType>` variant of the interfaces from aten, to reduce binary size.

Differential Revision: [D33102276](https://our.internmc.facebook.com/intern/diff/D33102276/)

[ghstack-poisoned]
  • Loading branch information
zhxchen17 committed Jan 8, 2022
2 parents 00f1ad8 + 7092310 commit 9553998
Show file tree
Hide file tree
Showing 72 changed files with 757 additions and 252 deletions.
186 changes: 186 additions & 0 deletions .github/scripts/syncbranches.py
@@ -0,0 +1,186 @@
#!/usr/bin/env python3

from collections import defaultdict
from datetime import datetime
from typing import cast, Any, Dict, List, Optional, Tuple, Union
import os


def _check_output(items: List[str], encoding: str = "utf-8") -> str:
from subprocess import check_output
return check_output(items).decode(encoding)


def fuzzy_list_to_dict(items: List[Tuple[str, str]]) -> Dict[str, List[str]]:
"""
Converts list to dict preserving elements with duplicate keys
"""
rc: Dict[str, List[str]] = defaultdict(lambda: [])
for (key, val) in items:
rc[key].append(val)
return dict(rc)


class GitCommit:
commit_hash: str
title: str
body: str
author: str
author_date: datetime
commit_date: Optional[datetime]

def __init__(self,
commit_hash: str,
author: str,
author_date: datetime,
title: str,
body: str,
commit_date: Optional[datetime] = None) -> None:
self.commit_hash = commit_hash
self.author = author
self.author_date = author_date
self.commit_date = commit_date
self.title = title
self.body = body

def __repr__(self) -> str:
return f"{self.title} ({self.commit_hash})"

def __contains__(self, item: Any) -> bool:
return item in self.body or item in self.title


def parse_fuller_format(lines: Union[str, List[str]]) -> GitCommit:
"""
Expect commit message generated using `--format=fuller --date=unix` format, i.e.:
commit <sha1>
Author: <author>
AuthorDate: <author date>
Commit: <committer>
CommitDate: <committer date>
<title line>
<full commit message>
"""
if isinstance(lines, str):
lines = lines.split("\n")
# TODO: Handle merge commits correctly
if len(lines) > 1 and lines[1].startswith("Merge:"):
del lines[1]
assert len(lines) > 7
assert lines[0].startswith("commit")
assert lines[1].startswith("Author: ")
assert lines[2].startswith("AuthorDate: ")
assert lines[3].startswith("Commit: ")
assert lines[4].startswith("CommitDate: ")
assert len(lines[5]) == 0
return GitCommit(commit_hash=lines[0].split()[1].strip(),
author=lines[1].split(":", 1)[1].strip(),
author_date=datetime.fromtimestamp(int(lines[2].split(":", 1)[1].strip())),
commit_date=datetime.fromtimestamp(int(lines[4].split(":", 1)[1].strip())),
title=lines[6].strip(),
body="\n".join(lines[7:]),
)


class GitRepo:
def __init__(self, path: str, remote: str = "origin") -> None:
self.repo_dir = path
self.remote = remote

def _run_git(self, *args: Any) -> str:
return _check_output(["git", "-C", self.repo_dir] + list(args))

def revlist(self, revision_range: str) -> List[str]:
rc = self._run_git("rev-list", revision_range, "--", ".").strip()
return rc.split("\n") if len(rc) > 0 else []

def current_branch(self) -> str:
return self._run_git("symbolic-ref", "--short", "HEAD").strip()

def checkout(self, branch: str) -> None:
self._run_git('checkout', branch)

def show_ref(self, name: str) -> str:
refs = self._run_git('show-ref', '-s', name).strip().split('\n')
if not all(refs[i] == refs[0] for i in range(1, len(refs))):
raise RuntimeError(f"referce {name} is ambigous")
return refs[0]

def rev_parse(self, name: str) -> str:
return self._run_git('rev-parse', '--verify', name).strip()

def get_merge_base(self, from_ref: str, to_ref: str) -> str:
return self._run_git('merge-base', from_ref, to_ref).strip()

def patch_id(self, ref: Union[str, List[str]]) -> List[Tuple[str, str]]:
is_list = isinstance(ref, list)
if is_list:
if len(ref) == 0:
return []
ref = " ".join(ref)
rc = _check_output(['sh', '-c', f'git -C {self.repo_dir} show {ref}|git patch-id --stable']).strip()
return [cast(Tuple[str, str], x.split(" ", 1)) for x in rc.split("\n")]

def get_commit(self, ref: str) -> GitCommit:
return parse_fuller_format(self._run_git('show', '--format=fuller', '--date=unix', '--shortstat', ref))

def cherry_pick(self, ref: str) -> None:
self._run_git('cherry-pick', '-x', ref)

def compute_branch_diffs(self, from_branch: str, to_branch: str) -> Tuple[List[str], List[str]]:
"""
Returns list of commmits that are missing in each other branch since their merge base
Might be slow if merge base is between two branches is pretty far off
"""
from_ref = self.rev_parse(from_branch)
to_ref = self.rev_parse(to_branch)
merge_base = self.get_merge_base(from_ref, to_ref)
from_commits = self.revlist(f'{merge_base}..{from_ref}')
to_commits = self.revlist(f'{merge_base}..{to_ref}')
from_ids = fuzzy_list_to_dict(self.patch_id(from_commits))
to_ids = fuzzy_list_to_dict(self.patch_id(to_commits))
for patch_id in set(from_ids).intersection(set(to_ids)):
from_values = from_ids[patch_id]
to_values = to_ids[patch_id]
if len(from_values) != len(to_values):
# Eliminate duplicate commits+reverts from the list
while len(from_values) > 0 and len(to_values) > 0:
frc = self.get_commit(from_values.pop())
toc = self.get_commit(to_values.pop())
if frc.title != toc.title or frc.author_date != toc.author_date:
raise RuntimeError(f"Unexpected differences between {frc} and {toc}")
from_commits.remove(frc.commit_hash)
to_commits.remove(toc.commit_hash)
continue
for commit in from_values:
from_commits.remove(commit)
for commit in to_values:
to_commits.remove(commit)
return (from_commits, to_commits)

def cherry_pick_commits(self, from_branch: str, to_branch: str) -> None:
orig_branch = self.current_branch()
self.checkout(to_branch)
from_commits, to_commits = self.compute_branch_diffs(from_branch, to_branch)
if len(from_commits) == 0:
print("Nothing to do")
self.checkout(orig_branch)
return
for commit in reversed(from_commits):
self.cherry_pick(commit)
self.checkout(orig_branch)

def push(self, branch: str) -> None:
self._run_git("push", self.remote, branch)


if __name__ == '__main__':
repo_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
default_branch = 'master'
sync_branch = 'fbsync'
repo = GitRepo(repo_dir)
repo.cherry_pick_commits(sync_branch, default_branch)
repo.push(default_branch)
4 changes: 2 additions & 2 deletions .github/templates/common.yml.j2
Expand Up @@ -78,8 +78,8 @@ concurrency:
docker run --pull=never --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- name: Clean workspace
run: |
rm -rf "${GITHUB_WORKSPACE:?}/*"
rm -f ~/.ssh/authorized_keys
rm -rf "${GITHUB_WORKSPACE}"
mkdir "${GITHUB_WORKSPACE}"
- name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
uses: seemethere/add-github-ssh-key@v1
with:
Expand Down
1 change: 0 additions & 1 deletion .github/templates/linux_ci_workflow.yml.j2
Expand Up @@ -18,7 +18,6 @@ on:
branches:
- master
- release/*
- fbsync
{%- endif %}
workflow_dispatch:

Expand Down
1 change: 0 additions & 1 deletion .github/templates/macos_ci_workflow.yml.j2
Expand Up @@ -18,7 +18,6 @@ on:
branches:
- master
- release/*
- fbsync
{%- endif %}
workflow_dispatch:

Expand Down
1 change: 0 additions & 1 deletion .github/templates/windows_ci_workflow.yml.j2
Expand Up @@ -29,7 +29,6 @@ on:
branches:
- master
- release/*
- fbsync
{%- endif %}
workflow_dispatch:

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions .github/workflows/generated-docker-builds.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 4 additions & 5 deletions .github/workflows/generated-linux-bionic-py3.7-clang9.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions .github/workflows/generated-linux-docs-push.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 9553998

Please sign in to comment.