From 52a43df4ccc7481805a546b6b1fe7e537f3da5ef Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 01:12:41 +0300 Subject: [PATCH 1/7] brag: Add missing fields to the sample JSON file Signed-off-by: Oleh Prypin --- src/brag/server/sample.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/brag/server/sample.json b/src/brag/server/sample.json index 194ec637be905..dd1299ca70540 100644 --- a/src/brag/server/sample.json +++ b/src/brag/server/sample.json @@ -5,7 +5,8 @@ "num_pgs": 192, "num_mdss": 1, "num_osds": 1, - "num_bytes": 0, + "num_data_bytes": 0, + "num_bytes_total": 0, "num_pools": 3, "num_mons": 1, "num_objects": 0 From 8d4d2787e539bb7a66a647dcfa7b01502e31b9f1 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 00:51:50 +0300 Subject: [PATCH 2/7] brag: Assume there are 0 MDS instead of crashing when data is missing Signed-off-by: Oleh Prypin --- src/brag/client/ceph-brag | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag index ba785e37785b0..006f076c5c58a 100755 --- a/src/brag/client/ceph-brag +++ b/src/brag/client/ceph-brag @@ -245,7 +245,10 @@ def get_nums(): oj = json.loads(o) num_mons = len(oj['monmap']['mons']) num_osds = int(oj['osdmap']['osdmap']['num_in_osds']) - num_mdss = oj['mdsmap']['in'] + try: + num_mdss = oj['mdsmap']['in'] + except KeyError: + num_mdss = 0 pgmap = oj['pgmap'] num_pgs = pgmap['num_pgs'] From f231b801ccb87fa2ffe8bdc47f442455860cb0f9 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 02:48:45 +0300 Subject: [PATCH 3/7] brag: Replace dangerous uses of `is` operator It is an implementation detail that `is` happens to behave the same as `==` for low integers Signed-off-by: Oleh Prypin --- src/brag/client/ceph-brag | 26 +++++++++---------- src/brag/server/ceph_brag/controllers/root.py | 6 ++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag index 006f076c5c58a..86f8ca8c8b81b 100755 --- a/src/brag/client/ceph-brag +++ b/src/brag/client/ceph-brag @@ -211,12 +211,12 @@ def run_command(cmd): def get_uuid(): (rc,uid,e) = run_command(['ceph', 'config-key', 'get', CLUSTER_UUID_NAME]) - if rc is not 0: + if rc: #uuid is not yet set. uid = str(uuid.uuid4()) (rc, o, e) = run_command(['ceph', 'config-key', 'put', CLUSTER_UUID_NAME, uid]) - if rc is not 0: + if rc: raise RuntimeError("\'ceph config-key put\' failed -" + e) return uid @@ -239,7 +239,7 @@ def bytes_pretty_to_raw(byte_count, byte_scale): def get_nums(): (rc, o, e) = run_command(['ceph', '-s', '-f', 'json']) - if rc is not 0: + if rc: raise RuntimeError("\'ceph -s\' failed - " + e) oj = json.loads(o) @@ -256,7 +256,7 @@ def get_nums(): num_bytes_total = pgmap['bytes_total'] (rc, o, e) = run_command(['ceph', 'pg', 'dump', 'pools', '-f', 'json-pretty']) - if rc is not 0: + if rc: raise RuntimeError("\'ceph pg dump pools\' failed - " + e) pools = json.loads(o) @@ -277,7 +277,7 @@ def get_nums(): def get_crush_types(): (rc, o, e) = run_command(['ceph', 'osd', 'crush', 'dump']) - if rc is not 0: + if rc: raise RuntimeError("\'ceph osd crush dump\' failed - " + e) crush_dump = json.loads(o) @@ -306,7 +306,7 @@ def get_crush_types(): def get_osd_dump_info(): (rc, o, e) = run_command(['ceph', 'osd', 'dump', '-f', 'json']) - if rc is not 0: + if rc: raise RuntimeError("\'ceph osd dump\' failed - " + e) pool_meta = [] @@ -332,8 +332,8 @@ def get_sysinfo(max_osds): incr = lambda a,k: 1 if k not in a else a[k]+1 while count < max_osds: (rc, o, e) = run_command(['ceph', 'osd', 'metadata', str(count)]) - if rc is 0: - if osd_metadata_available is False: + if rc == 0: + if not osd_metadata_available: osd_metadata_available = True jmeta = json.loads(o) @@ -365,7 +365,7 @@ def get_sysinfo(max_osds): count = count + 1 sysinfo = {} - if osd_metadata_available is False: + if not osd_metadata_available: print >> sys.stderr, "'ceph osd metadata' is not available at all" return sysinfo @@ -387,7 +387,7 @@ def get_sysinfo(max_osds): def get_ownership_info(): (rc, o, e) = run_command(['ceph', 'config-key', 'get', CLUSTER_OWNERSHIP_NAME]) - if rc is not 0: + if rc: return {} return ast.literal_eval(o) @@ -478,7 +478,7 @@ def publish(): if verbose: print "PUT " + str(url) + " : " + str(data) req = requests.put(url, data=data) - if req.status_code is not 201: + if req.status_code != 201: print >> sys.stderr, "Failed to publish, server responded with code " + str(req.status_code) print >> sys.stderr, req.text return 1 @@ -507,7 +507,7 @@ def unpublish(): params = {'uuid':uuid} req = requests.delete(url, params=params) - if req.status_code is not 200: + if req.status_code != 200: print >> sys.stderr, "Failed to unpublish, server responsed with code " + str(req.status_code) return 1 @@ -518,7 +518,7 @@ def main(): global verbose verbose = True sys.argv.pop(1) - if len(sys.argv) is 1: + if len(sys.argv) == 1: print output_json()[0] return 0 if sys.argv[1] == 'update-metadata': diff --git a/src/brag/server/ceph_brag/controllers/root.py b/src/brag/server/ceph_brag/controllers/root.py index a52f7f971157a..6ca46c8c081a1 100644 --- a/src/brag/server/ceph_brag/controllers/root.py +++ b/src/brag/server/ceph_brag/controllers/root.py @@ -11,13 +11,13 @@ def fail(self, status_code=200, msg="OK"): @expose('json') def get(self, *args, **kwargs): - if len(args) is 0: + if len(args) == 0: #return the list of uuids try: result = db.get_uuids() except Exception as e: return self.fail(500, msg="Internal Server Error") - elif len(args) is 1 or len(args) is 2 and args[1] == '': + elif len(args) == 1 or len(args) == 2 and args[1] == '': #/uuid try: result = db.get_versions(args[0]) @@ -26,7 +26,7 @@ def get(self, *args, **kwargs): if result is None: return self.fail(400, msg="Invalid UUID") - elif len(args) is 2 or len(args) is 3 and args[2] == '': + elif len(args) == 2 or len(args) == 3 and args[2] == '': #/uuid/version_number try: result = db.get_brag(args[0], args[1]) From edac38e21088e5326c88ce934858d38fc517a291 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 02:53:43 +0300 Subject: [PATCH 4/7] brag: Use print_function for Python 3 compatibility Signed-off-by: Oleh Prypin --- src/brag/client/ceph-brag | 75 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag index 86f8ca8c8b81b..df48e18379afd 100755 --- a/src/brag/client/ceph-brag +++ b/src/brag/client/ceph-brag @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + import subprocess import uuid import re @@ -200,9 +202,13 @@ class Counter(dict): return result +def print_stderr(*args, **kwargs): + kwargs.setdefault('file', sys.stderr) + print(*args, **kwargs) + def run_command(cmd): if verbose: - print "run_command: " + str(cmd) + print_stderr("run_command: " + str(cmd)) child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (o, e) = child.communicate() @@ -366,7 +372,7 @@ def get_sysinfo(max_osds): sysinfo = {} if not osd_metadata_available: - print >> sys.stderr, "'ceph osd metadata' is not available at all" + print_stderr("'ceph osd metadata' is not available at all") return sysinfo def jsonify(type_count, name, type_name): @@ -413,29 +419,30 @@ def output_json(): return json.dumps(out, indent=2, separators=(',', ': ')), url def describe_usage(): - print >> sys.stderr, "Usage:" - print >> sys.stderr, "======\n" - - print >> sys.stderr, sys.argv[0] + " [-v|--verbose] [ [command-options]]\n" - print >> sys.stderr, "without any option, shows the data to be published and do nothing" - print >> sys.stderr, "" - print >> sys.stderr, "-v|--verbose: toggle verbose output on stdout" - print >> sys.stderr, "" - print >> sys.stderr, "commands:" - print >> sys.stderr, "publish - publish the brag report to the server" - print >> sys.stderr, "update-metadata - Update" - print >> sys.stderr, " ownership information for bragging" - print >> sys.stderr, "clear-metadata - Clear information set by update-metadata" - print >> sys.stderr, "unpublish --yes-i-am-shy - delete the brag report from the server" - print >> sys.stderr, "" - - print >> sys.stderr, "update-metadata options:" - print >> sys.stderr, "--name= - Name of the cluster" - print >> sys.stderr, "--organization= - Name of the organization" - print >> sys.stderr, "--email= - Email contact address" - print >> sys.stderr, "--description= - Reporting use-case" - print >> sys.stderr, "--url= - The URL that is used to publish and unpublish" - print >> sys.stderr, "" + print_stderr("Usage:") + print_stderr("======") + print_stderr() + print_stderr(sys.argv[0] + " [-v|--verbose] [ [command-options]]") + print_stderr() + print_stderr("without any option, shows the data to be published and do nothing") + print_stderr() + print_stderr("-v|--verbose: toggle verbose output on stdout") + print_stderr() + print_stderr("commands:") + print_stderr("publish - publish the brag report to the server") + print_stderr("update-metadata - Update") + print_stderr(" ownership information for bragging") + print_stderr("clear-metadata - Clear information set by update-metadata") + print_stderr("unpublish --yes-i-am-shy - delete the brag report from the server") + print_stderr() + + print_stderr("update-metadata options:") + print_stderr("--name= - Name of the cluster") + print_stderr("--organization= - Name of the organization") + print_stderr("--email= - Email contact address") + print_stderr("--description= - Reporting use-case") + print_stderr("--url= - The URL that is used to publish and unpublish") + print_stderr() def update_metadata(): info = {} @@ -456,7 +463,7 @@ def update_metadata(): if k in possibles: info[k] = v else: - print >> sys.stderr, "Unexpect option --" + k + print_stderr("Unexpect option --" + k) describe_usage() return 22 @@ -472,22 +479,22 @@ def clear_metadata(): def publish(): data, url = output_json() if url is None: - print >> sys.stderr, "Cannot publish until a URL is set using update-metadata" + print_stderr("Cannot publish until a URL is set using update-metadata") return 1 if verbose: - print "PUT " + str(url) + " : " + str(data) + print_stderr("PUT " + str(url) + " : " + str(data)) req = requests.put(url, data=data) if req.status_code != 201: - print >> sys.stderr, "Failed to publish, server responded with code " + str(req.status_code) - print >> sys.stderr, req.text + print_stderr("Failed to publish, server responded with code " + str(req.status_code)) + print_stderr(req.text) return 1 return 0 def unpublish(): if len(sys.argv) <= 2 or sys.argv[2] != '--yes-i-am-shy': - print >> sys.stderr, "unpublish should be followed by --yes-i-am-shy" + print_stderr("unpublish should be followed by --yes-i-am-shy") return 22 fail = False @@ -500,7 +507,7 @@ def unpublish(): fail = True if fail: - print >> sys.stderr, "URL is not updated yet" + print_stderr("URL is not updated yet") return 1 uuid = get_uuid() @@ -508,7 +515,7 @@ def unpublish(): params = {'uuid':uuid} req = requests.delete(url, params=params) if req.status_code != 200: - print >> sys.stderr, "Failed to unpublish, server responsed with code " + str(req.status_code) + print_stderr("Failed to unpublish, server responsed with code " + str(req.status_code)) return 1 return 0 @@ -519,7 +526,7 @@ def main(): verbose = True sys.argv.pop(1) if len(sys.argv) == 1: - print output_json()[0] + print(output_json()[0]) return 0 if sys.argv[1] == 'update-metadata': return update_metadata() From 9924da025898539ab678383332bc46d3fb79b863 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 02:54:42 +0300 Subject: [PATCH 5/7] brag: Fix relative import for Python 3 compatibility Signed-off-by: Oleh Prypin --- src/brag/server/ceph_brag/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/brag/server/ceph_brag/model/__init__.py b/src/brag/server/ceph_brag/model/__init__.py index 24e34828ce6b3..ad1a942245f0f 100644 --- a/src/brag/server/ceph_brag/model/__init__.py +++ b/src/brag/server/ceph_brag/model/__init__.py @@ -1,6 +1,6 @@ from sqlalchemy import create_engine from pecan import conf # noqa -from db import Session, Base +from .db import Session, Base import sys def create_from_conf(): From 56287392d86c916e0d9ae03fa8aaf384ab22f909 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 02:55:22 +0300 Subject: [PATCH 6/7] brag: Make usage of bytes vs unicode compatible with Python 3 Signed-off-by: Oleh Prypin --- src/brag/client/ceph-brag | 2 ++ src/brag/server/ceph_brag/controllers/root.py | 2 +- .../server/ceph_brag/tests/test_functional.py | 16 ++++++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag index df48e18379afd..32a52c41fe61c 100755 --- a/src/brag/client/ceph-brag +++ b/src/brag/client/ceph-brag @@ -212,6 +212,8 @@ def run_command(cmd): child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (o, e) = child.communicate() + o = o.decode('utf-8', 'ignore') + e = e.decode('utf-8', 'ignore') return (child.returncode, o, e) diff --git a/src/brag/server/ceph_brag/controllers/root.py b/src/brag/server/ceph_brag/controllers/root.py index 6ca46c8c081a1..56ce1449752c2 100644 --- a/src/brag/server/ceph_brag/controllers/root.py +++ b/src/brag/server/ceph_brag/controllers/root.py @@ -43,7 +43,7 @@ def get(self, *args, **kwargs): @expose(content_type='application/json') def put(self, *args, **kwargs): try: - db.put_new_version(request.body) + db.put_new_version(request.body.decode('utf-8')) except ValueError as ve: return self.fail(status_code=422, msg="Improper payload : " + str(ve)) except KeyError as ke: diff --git a/src/brag/server/ceph_brag/tests/test_functional.py b/src/brag/server/ceph_brag/tests/test_functional.py index 292338c276674..03436cbdef063 100644 --- a/src/brag/server/ceph_brag/tests/test_functional.py +++ b/src/brag/server/ceph_brag/tests/test_functional.py @@ -10,27 +10,27 @@ def test_1_get_invalid_url_format(self): assert response.status_int == 400 def test_2_put(self): - with open ("sample.json", "r") as myfile: - data=myfile.read().replace('\n', '') + with open("sample.json", "rb") as myfile: + data = myfile.read().replace(b'\n', b'') response = self.app.request('/', method='PUT', body=data) assert response.status_int == 201 def test_3_put_invalid_json(self): - response = self.app.request('/', method='PUT', body='{asdfg', expect_errors=True) + response = self.app.request('/', method='PUT', body=b'{asdfg', expect_errors=True) assert response.status_int == 422 def test_4_put_invalid_entries_1(self): - response = self.app.request('/', method='PUT', body='{}', expect_errors=True) + response = self.app.request('/', method='PUT', body=b'{}', expect_errors=True) assert response.status_int == 422 def test_5_put_incomplete_json(self): - response = self.app.request('/', method='PUT', body='{\"uuid\":\"adfs-12312ad\"}', + response = self.app.request('/', method='PUT', body=b'{"uuid":"adfs-12312ad"}', expect_errors=True) assert response.status_int == 422 def test_6_get(self): response = self.app.get('/') - js = json.loads(response.body) + js = json.loads(response.body.decode('utf-8')) for entry in js: ci = entry break @@ -44,7 +44,7 @@ def test_7_get_invalid_uuid(self): def test_8_get_invalid_version(self): response = self.app.get('/') - js = json.loads(response.body) + js = json.loads(response.body.decode('utf-8')) for entry in js: ci = entry break @@ -62,7 +62,7 @@ def test_91_delete_wrong_uuid(self): def test_92_delete(self): response = self.app.get('/') - js = json.loads(response.body) + js = json.loads(response.body.decode('utf-8')) for entry in js: response = self.app.delete('/?uuid='+entry['uuid']) assert response.status_int == 200 From 1b68935e233710373a0f141dbe329afb783c8946 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 30 Jun 2016 16:09:47 +0300 Subject: [PATCH 7/7] brag: Use Counter from standard library if available Signed-off-by: Oleh Prypin --- src/brag/client/ceph-brag | 354 +++++++++++++++++++------------------- 1 file changed, 179 insertions(+), 175 deletions(-) diff --git a/src/brag/client/ceph-brag b/src/brag/client/ceph-brag index 32a52c41fe61c..8a0485934f872 100755 --- a/src/brag/client/ceph-brag +++ b/src/brag/client/ceph-brag @@ -11,7 +11,6 @@ import ast import requests from operator import itemgetter from heapq import nlargest -from itertools import repeat, ifilter CLUSTER_UUID_NAME='cluster-uuid' @@ -20,186 +19,191 @@ CLUSTER_OWNERSHIP_NAME='cluster-ownership' verbose = False -class Counter(dict): - '''Dict subclass for counting hashable objects. Sometimes called a bag - or multiset. Elements are stored as dictionary keys and their counts - are stored as dictionary values. +try: + from collections import Counter +except ImportError: + from itertools import repeat, ifilter - >>> Counter('zyzygy') - Counter({'y': 3, 'z': 2, 'g': 1}) + class Counter(dict): + '''Dict subclass for counting hashable objects. Sometimes called a bag + or multiset. Elements are stored as dictionary keys and their counts + are stored as dictionary values. - ''' - - def __init__(self, iterable=None, **kwds): - '''Create a new, empty Counter object. And if given, count elements - from an input iterable. Or, initialize the count from another mapping - of elements to their counts. - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping - >>> c = Counter(a=4, b=2) # a new counter from keyword args + >>> Counter('zyzygy') + Counter({'y': 3, 'z': 2, 'g': 1}) ''' - self.update(iterable, **kwds) - - def __missing__(self, key): - return 0 - - def most_common(self, n=None): - '''List the n most common elements and their counts from the most - common to the least. If n is None, then list all element counts. - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('r', 2), ('b', 2)] - - ''' - if n is None: - return sorted(self.iteritems(), key=itemgetter(1), reverse=True) - return nlargest(n, self.iteritems(), key=itemgetter(1)) - - def elements(self): - '''Iterator over elements repeating each as many times as its count. - - >>> c = Counter('ABCABC') - >>> sorted(c.elements()) - ['A', 'A', 'B', 'B', 'C', 'C'] - - If an element's count has been set to zero or is a negative number, - elements() will ignore it. - - ''' - for elem, count in self.iteritems(): - for _ in repeat(None, count): - yield elem - - # Override dict methods where the meaning changes for Counter objects. - - @classmethod - def fromkeys(cls, iterable, v=None): - raise NotImplementedError( - 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - - def update(self, iterable=None, **kwds): - '''Like dict.update() but add counts instead of replacing them. - - Source can be an iterable, a dictionary, or another Counter instance. - - >>> c = Counter('which') - >>> c.update('witch') # add elements from another iterable - >>> d = Counter('watch') - >>> c.update(d) # add elements from another counter - >>> c['h'] # four 'h' in which, witch, and watch - 4 - ''' - if iterable is not None: - if hasattr(iterable, 'iteritems'): - if self: - self_get = self.get - for elem, count in iterable.iteritems(): - self[elem] = self_get(elem, 0) + count + def __init__(self, iterable=None, **kwds): + '''Create a new, empty Counter object. And if given, count elements + from an input iterable. Or, initialize the count from another mapping + of elements to their counts. + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping + >>> c = Counter(a=4, b=2) # a new counter from keyword args + + ''' + self.update(iterable, **kwds) + + def __missing__(self, key): + return 0 + + def most_common(self, n=None): + '''List the n most common elements and their counts from the most + common to the least. If n is None, then list all element counts. + + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('r', 2), ('b', 2)] + + ''' + if n is None: + return sorted(self.iteritems(), key=itemgetter(1), reverse=True) + return nlargest(n, self.iteritems(), key=itemgetter(1)) + + def elements(self): + '''Iterator over elements repeating each as many times as its count. + + >>> c = Counter('ABCABC') + >>> sorted(c.elements()) + ['A', 'A', 'B', 'B', 'C', 'C'] + + If an element's count has been set to zero or is a negative number, + elements() will ignore it. + + ''' + for elem, count in self.iteritems(): + for _ in repeat(None, count): + yield elem + + # Override dict methods where the meaning changes for Counter objects. + + @classmethod + def fromkeys(cls, iterable, v=None): + raise NotImplementedError( + 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') + + def update(self, iterable=None, **kwds): + '''Like dict.update() but add counts instead of replacing them. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.update('witch') # add elements from another iterable + >>> d = Counter('watch') + >>> c.update(d) # add elements from another counter + >>> c['h'] # four 'h' in which, witch, and watch + 4 + + ''' + if iterable is not None: + if hasattr(iterable, 'iteritems'): + if self: + self_get = self.get + for elem, count in iterable.iteritems(): + self[elem] = self_get(elem, 0) + count + else: + dict.update(self, iterable) # fast path when counter is empty else: - dict.update(self, iterable) # fast path when counter is empty - else: - self_get = self.get - for elem in iterable: - self[elem] = self_get(elem, 0) + 1 - if kwds: - self.update(kwds) - - def copy(self): - 'Like dict.copy() but returns a Counter instance instead of a dict.' - return Counter(self) - - def __delitem__(self, elem): - 'Like dict.__delitem__() but does not raise KeyError for missing values.' - if elem in self: - dict.__delitem__(self, elem) - - def __repr__(self): - if not self: - return '%s()' % self.__class__.__name__ - items = ', '.join(map('%r: %r'.__mod__, self.most_common())) - return '%s({%s})' % (self.__class__.__name__, items) - - # Multiset-style mathematical operations discussed in: - # Knuth TAOCP Volume II section 4.6.3 exercise 19 - # and at http://en.wikipedia.org/wiki/Multiset - # - # Outputs guaranteed to only include positive counts. - # - # To strip negative and zero counts, add-in an empty counter: - # c += Counter() - - def __add__(self, other): - '''Add counts from two counters. - - >>> Counter('abbb') + Counter('bcc') - Counter({'b': 4, 'c': 2, 'a': 1}) - - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] + other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __sub__(self, other): - ''' Subtract count, but keep only results with positive counts. - - >>> Counter('abbbc') - Counter('bccd') - Counter({'b': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - result = Counter() - for elem in set(self) | set(other): - newcount = self[elem] - other[elem] - if newcount > 0: - result[elem] = newcount - return result - - def __or__(self, other): - '''Union is the maximum of value in either of the input counters. - - >>> Counter('abbb') | Counter('bcc') - Counter({'b': 3, 'c': 2, 'a': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _max = max - result = Counter() - for elem in set(self) | set(other): - newcount = _max(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result - - def __and__(self, other): - ''' Intersection is the minimum of corresponding counts. - - >>> Counter('abbb') & Counter('bcc') - Counter({'b': 1}) - - ''' - if not isinstance(other, Counter): - return NotImplemented - _min = min - result = Counter() - if len(self) < len(other): - self, other = other, self - for elem in ifilter(self.__contains__, other): - newcount = _min(self[elem], other[elem]) - if newcount > 0: - result[elem] = newcount - return result + self_get = self.get + for elem in iterable: + self[elem] = self_get(elem, 0) + 1 + if kwds: + self.update(kwds) + + def copy(self): + 'Like dict.copy() but returns a Counter instance instead of a dict.' + return Counter(self) + + def __delitem__(self, elem): + 'Like dict.__delitem__() but does not raise KeyError for missing values.' + if elem in self: + dict.__delitem__(self, elem) + + def __repr__(self): + if not self: + return '%s()' % self.__class__.__name__ + items = ', '.join(map('%r: %r'.__mod__, self.most_common())) + return '%s({%s})' % (self.__class__.__name__, items) + + # Multiset-style mathematical operations discussed in: + # Knuth TAOCP Volume II section 4.6.3 exercise 19 + # and at http://en.wikipedia.org/wiki/Multiset + # + # Outputs guaranteed to only include positive counts. + # + # To strip negative and zero counts, add-in an empty counter: + # c += Counter() + + def __add__(self, other): + '''Add counts from two counters. + + >>> Counter('abbb') + Counter('bcc') + Counter({'b': 4, 'c': 2, 'a': 1}) + + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] + other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __sub__(self, other): + ''' Subtract count, but keep only results with positive counts. + + >>> Counter('abbbc') - Counter('bccd') + Counter({'b': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem in set(self) | set(other): + newcount = self[elem] - other[elem] + if newcount > 0: + result[elem] = newcount + return result + + def __or__(self, other): + '''Union is the maximum of value in either of the input counters. + + >>> Counter('abbb') | Counter('bcc') + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _max = max + result = Counter() + for elem in set(self) | set(other): + newcount = _max(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result + + def __and__(self, other): + ''' Intersection is the minimum of corresponding counts. + + >>> Counter('abbb') & Counter('bcc') + Counter({'b': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + _min = min + result = Counter() + if len(self) < len(other): + self, other = other, self + for elem in ifilter(self.__contains__, other): + newcount = _min(self[elem], other[elem]) + if newcount > 0: + result[elem] = newcount + return result def print_stderr(*args, **kwargs):