Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Profiling Sync
  • Loading branch information
@fao89
fao89 committed Nov 22, 2019
1 parent 5cbcc49 commit f3f0790
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 1 deletion.
4 changes: 4 additions & 0 deletions pulp_rpm/app/tasks/synchronizing.py
View file
Expand Up @@ -123,6 +123,10 @@ def synchronize(remote_pk, repository_pk):
deferred_download = (remote.policy != Remote.IMMEDIATE) # Interpret download policy

treeinfo = get_treeinfo_data(remote)
import tracemalloc
from pulp_rpm.tests.performance.utils import TakeSnapshot
tracemalloc.start(25)
TakeSnapshot().start()

This comment has been minimized.

Sign in to view

Copy link
@fao89

fao89 Nov 22, 2019

Author Member

right before instantiating a RpmDeclarativeVersion
if treeinfo:
treeinfo["repositories"] = {}
for repodata in set(treeinfo["download"]["repodatas"]):
Expand Down
1 change: 1 addition & 0 deletions pulp_rpm/tests/functional/constants.py
View file
Expand Up @@ -258,3 +258,4 @@
CENTOS8_APPSTREAM_URL = "http://mirror.linux.duke.edu/pub/centos/8/AppStream/x86_64/os/"
CENTOS8_BASEOS_URL = "http://mirror.linux.duke.edu/pub/centos/8/BaseOS/x86_64/os/"
EPEL7_URL = "https://dl.fedoraproject.org/pub/epel/7/x86_64/"
RHEL7_URL = "http://cdn.stage.redhat.com/content/dist/rhel/server/7/7Server/x86_64/os/"
5 changes: 5 additions & 0 deletions pulp_rpm/tests/performance/test_sync.py
View file
Expand Up @@ -24,6 +24,7 @@
CENTOS8_BASEOS_URL,
CENTOS8_KICKSTART_APP_URL,
CENTOS8_KICKSTART_BASEOS_URL,
RHEL7_URL,
)
from pulp_rpm.tests.functional.utils import gen_rpm_remote
from pulp_rpm.tests.functional.utils import set_up_module as setUpModule # noqa:F401
Expand Down Expand Up @@ -148,3 +149,7 @@ def test_centos8_kickstart_baseos_on_demand(self):
def test_centos8_kickstart_appstream_on_demand(self):
"""Kickstart Sync CentOS 8 AppStream."""
self.rpm_sync(url=CENTOS8_KICKSTART_APP_URL)

def test_rhel7_on_demand(self):
"""Sync RHEL 7."""
self.rpm_sync(url=RHEL7_URL)
31 changes: 30 additions & 1 deletion pulp_rpm/tests/performance/utils.py
View file
@@ -1,6 +1,12 @@
from collections import namedtuple
import pickle
import psutil

import gc
import os
import signal
import threading
import time
import tracemalloc

START = 'START'
END = 'END'
Expand All @@ -14,3 +20,26 @@ def get_consumed_ram():
for proc in psutil.process_iter():
info.update({proc.name(): (proc.memory_info().rss, proc.memory_info().vms)})
return info


class TakeSnapshot(threading.Thread):
daemon = True

def run(self):
if hasattr(signal, 'pthread_sigmask'):
# Available on UNIX with Python 3.3+
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
counter = 1
while True:
time.sleep(30)

This comment has been minimized.

Sign in to view

Copy link
@fao89

fao89 Nov 22, 2019

Author Member

taking snapshots every 30 seconds
filename = ("/tmp/tracemalloc-%d-%04d.pickle"
% (os.getpid(), counter))
print("Write snapshot into %s..." % filename)
gc.collect()
snapshot = tracemalloc.take_snapshot()
with open(filename, "wb") as fp:
# Pickle version 2 can be read by Python 2 and Python 3
pickle.dump(snapshot, fp, 2)
snapshot = None
print("Snapshot written into %s" % filename)
counter += 1

1 comment on commit f3f0790

@fao89
Copy link
Member Author

@fao89 fao89 commented on f3f0790 Nov 22, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After running the sync test for RHEL 7,
I ran this script:

import linecache
import os
import tracemalloc

snapshots = []

traces = [t for t in os.listdir("/tmp") if t.startswith("tracemalloc-")]
traces = sorted(traces, key=lambda t: int(t.split(".")[-2].split("-")[-1]))

def display_top(snapshot, key_type='lineno'):
    snapshot = snapshot.filter_traces((
        tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
        tracemalloc.Filter(False, "<unknown>"),
    ))
    top_stats = snapshot.statistics(key_type=key_type, cumulative=True)

    for index, stat in enumerate(top_stats, 1):
        frame = stat.traceback[0]
        # replace "/path/to/module/file.py" with "module/file.py"
        filename = os.sep.join(frame.filename.split(os.sep)[-2:])
        if (stat.size / 1024 / 1024) < 1:
            break
        print("#%s: %s:%s: %.1f KiB"
              % (index, filename, frame.lineno, stat.size / 1024))
        line = linecache.getline(frame.filename, frame.lineno).strip()
        if line:
            print('    %s' % line)

    other = top_stats[index:]
    if other:
        size = sum(stat.size for stat in other)
        print("%s other: %.1f MiB" % (len(other), size / 1024 / 1024))
    total = sum(stat.size for stat in top_stats)
    print("Total allocated size: %.1f MiB" % (total / 1024 / 1024))

print("\nLast Snapshot") 
last_trace = "/tmp/" + traces[-1]
snapshot = tracemalloc.Snapshot.load(last_trace)
display_top(snapshot)

sometimes the last snapshot is corrupted because it was being saved when proccess got killed

Please sign in to comment.