Merge branch 'release/0.3.3'

.gitignore

@@ -23,5 +23,6 @@ TEST-*.xml
 
 # documentation
 docs/_build/
+docs/drfapi/
 
 /static_files

docs/conf.py

@@ -258,3 +258,15 @@
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'nodeconductor.server.test_settings'
+from django.conf import settings
+settings.INSTALLED_APPS = [app for app in settings.INSTALLED_APPS if not app.endswith('tests')]
+settings.BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+from django.core.wsgi import get_wsgi_application
+get_wsgi_application()
+
+from nodeconductor.core.management.commands.drfdocs import Command
+Command().handle('nodeconductor_killbill', path='docs/drfapi')
+

docs/index.rst

@@ -20,6 +20,15 @@ API
 
    api
 
+Endpoints
+---------
+
+.. toctree::
+   :maxdepth: 1
+
+   drfapi/index
+
+
 License
 -------
 

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-
+import sys
 from setuptools import setup, find_packages
 
 
@@ -8,15 +8,32 @@
 ]
 
 install_requires = [
-    'nodeconductor>=0.79.0',
+    'nodeconductor>0.91.0',
     'lxml>=3.2',
     'xhtml2pdf>=0.0.6',
 ]
 
 
+# RPM installation does not need oslo, cliff and stevedore libs -
+# they are required only for installation with setuptools
+try:
+    action = sys.argv[1]
+except IndexError:
+    pass
+else:
+    if action in ['develop', 'install', 'test', 'bdist_egg']:
+        install_requires += [
+            'cliff==1.7.0',
+            'oslo.config==1.4.0',
+            'oslo.i18n==1.0.0',
+            'oslo.utils==1.0.0',
+            'stevedore==1.0.0',
+        ]
+
+
 setup(
     name='nodeconductor-killbill',
-    version='0.3.2',
+    version='0.3.3',
     author='OpenNode Team',
     author_email='info@opennodecloud.com',
     url='http://nodeconductor.com',

src/nodeconductor_killbill/apps.py

@@ -2,17 +2,17 @@
 from django.db.models import signals
 from django_fsm.signals import post_transition
 
-from nodeconductor.structure import models as structure_models
-from nodeconductor.core.handlers import preserve_fields_before_update
-
-from . import handlers
-
 
 class KillBillConfig(AppConfig):
     name = 'nodeconductor_killbill'
     verbose_name = "NodeConductor KillBill"
 
     def ready(self):
+        from nodeconductor.structure import models as structure_models
+        from nodeconductor.core.handlers import preserve_fields_before_update
+
+        from . import handlers
+
         Invoice = self.get_model('Invoice')
 
         signals.post_save.connect(

src/nodeconductor_killbill/backend.py

@@ -1,6 +1,7 @@
 import re
 import json
 import requests
+import urlparse
 import logging
 
 from datetime import datetime, timedelta
@@ -139,6 +140,7 @@ def __init__(self, api_url=None, username=None, password=None, api_key=None, api
             auth=(username, password))
 
         self.accounts = KillBill.Account(self.credentials)
+        self.bundles = KillBill.Bundle(self.credentials)
         self.catalog = KillBill.Catalog(self.credentials)
         self.invoices = KillBill.Invoice(self.credentials)
         self.subscriptions = KillBill.Subscription(self.credentials)
@@ -162,6 +164,9 @@ def _parse_invoice_data(self, raw_invoice):
         for item in raw_invoice['items']:
             if item['amount']:
                 fields = self.get_subscription_fields(item['subscriptionId'])
+                if not fields:
+                    logger.warn('Missing metadata, skipping invoice item %s' % item['invoiceItemId'])
+                    continue
                 invoice['items'].append(dict(
                     backend_id=item['invoiceItemId'],
                     name=item['usageName'] or item['description'],
@@ -210,6 +215,17 @@ def add_subscription(self, client_id, resource):
         # killbill server must be run in test mode for these tricks
         # -Dorg.killbill.server.test.mode=true
 
+        try:
+            subscriptions = self.bundles.list(externalKey=resource.uuid.hex)['subscriptions']
+            subscription_id = subscriptions[0]['subscriptionId']
+            self.update_subscription_fields(
+                subscription_id,
+                resource_name=resource.full_name,
+                project_name=resource.project.full_name)
+            return subscription_id
+        except NotFoundKillBillError:
+            pass
+
         content_type = ContentType.objects.get_for_model(resource)
         product_name = self._get_product_name_for_content_type(content_type)
         subscription = self.subscriptions.create(
@@ -222,7 +238,7 @@ def add_subscription(self, client_id, resource):
 
         self.set_subscription_fields(
             subscription['subscriptionId'],
-            resource_name=resource.name,
+            resource_name=resource.full_name,
             project_name=resource.project.full_name)
 
         return subscription['subscriptionId']
@@ -290,10 +306,15 @@ def set_subscription_fields(self, subscription_id, **data):
             data=json.dumps(fields))
 
     def update_subscription_fields(self, subscription_id, **data):
-        fields = self.subscriptions.get(subscription_id, 'customFields')
-        flist = ','.join(f['customFieldId'] for f in fields if f['name'] in data)
-        self.subscriptions._object_query(
-            subscription_id, 'customFields', method='DELETE', customFieldList=flist)
+        try:
+            fields = self.subscriptions.get(subscription_id, 'customFields')
+        except NotFoundKillBillError:
+            pass
+        else:
+            flist = ','.join(f['customFieldId'] for f in fields if f['name'] in data)
+            self.subscriptions._object_query(
+                subscription_id, 'customFields', method='DELETE', customFieldList=flist)
+
         self.set_subscription_fields(subscription_id, **data)
 
     def propagate_pricelist(self):
@@ -313,7 +334,7 @@ def propagate_pricelist(self):
 
             usages = E.usages()
             for priceitem in DefaultPriceListItem.objects.filter(resource_content_type=cid):
-                usage_name = re.sub(r'[\s:;,+%&$@/]+', '', "{}-{}".format(priceitem.item_type, priceitem.key))
+                usage_name = re.sub(r'[\s:;,+%&$@/]+', '', "{}-{}-{}".format(priceitem.item_type, priceitem.key, cid))
                 unit_name = UNIT_PREFIX + usage_name
                 usage = E.usage(
                     E.billingPeriod('MONTHLY'),
@@ -425,7 +446,8 @@ def request(self, url, method='GET', data=None, verify=False, **kwargs):
                 headers['Content-Type'] = self.type
                 headers['X-Killbill-CreatedBy'] = 'NodeConductor'
 
-            url = url if url.startswith(self.api_url) else self.api_url + url
+            if not urlparse.urlparse(url).netloc:
+                url = self.api_url + url
 
             try:
                 response = getattr(requests, method.lower())(
@@ -481,6 +503,9 @@ def request(self, url, method='GET', data=None, verify=False, **kwargs):
     class Account(BaseResource):
         path = 'accounts'
 
+    class Bundle(BaseResource):
+        path = 'bundles'
+
     class Catalog(BaseResource):
         path = 'catalog'
         type = 'application/xml'

src/nodeconductor_killbill/extension.py

@@ -22,6 +22,11 @@ class Settings:
             'INVOICE': {
                 'logo': 'gcloud-logo.png',
                 'company': 'OpenNode',
+                'address': 'Lille 4-205',
+                'country': 'Estonia',
+                'email': 'info@opennodecloud.com',
+                'postal': '80041',
+                'phone': '(+372) 555-55-55',
                 'bank': 'American Bank',
                 'account': '123456789',
             },

src/nodeconductor_killbill/handlers.py

@@ -58,22 +58,32 @@ def update_resource_name(sender, instance, created=False, **kwargs):
     if not created and instance.billing_backend_id and instance.name != instance._old_values['name']:
         backend = KillBillBackend()
         backend.update_subscription_fields(
-            instance.billing_backend_id, resource_name=instance.name)
+            instance.billing_backend_id, resource_name=instance.full_name)
 
 
 def update_project_name(sender, instance, created=False, **kwargs):
     if not created and instance.tracker.has_changed('name'):
         backend = KillBillBackend()
         for model in PaidResource.get_all_models():
-            for resource in model.objects.filter(project=instance):
-                backend.update_subscription_fields(
-                    resource.billing_backend_id, project_name=resource.project.full_name)
+            for resource in model.objects.exclude(billing_backend_id=None).filter(project=instance):
+                try:
+                    backend.update_subscription_fields(
+                        resource.billing_backend_id, project_name=resource.project.full_name)
+                except KillBillError as e:
+                    logger.error(
+                        "Failed to update project name in KillBill for resource %s: %s",
+                        resource, e)
 
 
 def update_project_group_name(sender, instance, created=False, **kwargs):
     if not created and instance.tracker.has_changed('name'):
         backend = KillBillBackend()
         for model in PaidResource.get_all_models():
-            for resource in model.objects.filter(project__project_groups=instance):
-                backend.update_subscription_fields(
-                    resource.billing_backend_id, project_name=resource.project.full_name)
+            for resource in model.objects.exclude(billing_backend_id=None).filter(project__project_groups=instance):
+                try:
+                    backend.update_subscription_fields(
+                        resource.billing_backend_id, project_name=resource.project.full_name)
+                except KillBillError as e:
+                    logger.error(
+                        "Failed to update project group name in KillBill for resource %s: %s",
+                        resource, e)

src/nodeconductor_killbill/log.py

@@ -1,4 +1,4 @@
-from nodeconductor.logging.log import EventLogger, event_logger
+from nodeconductor.logging.loggers import EventLogger, event_logger
 
 from .models import Invoice
 

src/nodeconductor_killbill/migrations/0001_initial.py

@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from django.db import models, migrations
-import nodeconductor.logging.log
+import nodeconductor.logging.loggers
 import uuidfield.fields
 
 
@@ -28,6 +28,6 @@ class Migration(migrations.Migration):
             options={
                 'abstract': False,
             },
-            bases=(nodeconductor.logging.log.LoggableMixin, models.Model),
+            bases=(nodeconductor.logging.loggers.LoggableMixin, models.Model),
         ),
     ]

src/nodeconductor_killbill/models.py

@@ -1,6 +1,7 @@
+from __future__ import unicode_literals
+
 import os
 import logging
-import functools
 import collections
 import StringIO
 import xhtml2pdf.pisa as pisa
@@ -9,15 +10,14 @@
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.template.loader import render_to_string
-from django.utils.lru_cache import lru_cache
 from django.utils.encoding import python_2_unicode_compatible
 
 from nodeconductor.core import models as core_models
 from nodeconductor.cost_tracking.models import DefaultPriceListItem
-from nodeconductor.logging.log import LoggableMixin
+from nodeconductor.logging.loggers import LoggableMixin
 from nodeconductor.structure.models import Customer
 
-from .backend import UNIT_PREFIX, KillBillBackend, KillBillError
+from .backend import UNIT_PREFIX, KillBillBackend
 
 
 logger = logging.getLogger(__name__)
@@ -126,9 +126,17 @@ def generate_usage_pdf(self, invoice):
                 unit = ('GB/hour' if price_item.item_type == 'storage' else 'hour') + (
                     's' if usage > 1 else '')
 
-                item['name'] = price_item.name
-                item['usage'] = "{:.2f} {} x {:.2f} {}".format(
-                    usage, unit, price_item.value, item['currency'])
+                # XXX: black magic need to replace MBs to GBs for display of storage values
+                if price_item.item_type == 'storage' and 'MB' in price_item.name:
+                    from decimal import Decimal
+                    item['name'] = price_item.name.replace('MB', 'GB')
+                    usage /= 1024.0
+                    value = price_item.value * Decimal('1024.0')
+                else:
+                    item['name'] = price_item.name
+                    value = price_item.value
+                item['usage'] = "{:.3f} {} x {:.3f} {}".format(
+                    usage, unit, value, item['currency'])
 
             resource = item['resource']
             resources.setdefault(resource, {'items': [], 'amount': 0})

src/nodeconductor_killbill/tasks.py

@@ -21,14 +21,14 @@ def sync_pricelist():
     try:
         backend.propagate_pricelist()
     except KillBillError as e:
-        logger.error("Can't propagade pricelist to %s: %s", backend, e)
+        logger.error("Can't propagate pricelist to %s: %s", backend, e)
 
 
 @shared_task(name='nodeconductor.killbill.sync_invoices')
 def sync_invoices():
     customers = set()
     for model in PaidResource.get_all_models():
-        for resource in model.objects.all():
+        for resource in model.objects.exclude(billing_backend_id=''):
             customers.add(resource.customer)
 
     for customer in customers:
@@ -71,7 +71,7 @@ def update_today_usage_of_resource(resource_str):
                 "Can't update usage for resource %s which is not subscribed to backend", resource_str)
             return
 
-        numerical = ['storage', 'users']  # XXX: use consistent method for usage calculation
+        numerical = cs_backend.NUMERICAL
         content_type = ContentType.objects.get_for_model(resource)
 
         units = {
@@ -93,8 +93,11 @@ def update_today_usage_of_resource(resource_str):
                 except KeyError:
                     logger.error("Can't find price for usage item %s:%s", key, val)
 
-        kb_backend = KillBillBackend()
-        kb_backend.add_usage_data(resource, usage)
+        try:
+            kb_backend = KillBillBackend()
+            kb_backend.add_usage_data(resource, usage)
+        except KillBillError as e:
+            logger.error("Can't add usage for resource %s: %s", resource, e)
 
         resource.last_usage_update_time = timezone.now()
         resource.save(update_fields=['last_usage_update_time'])