Complete removal of vitrage-collector service.

- vitrage-graph will execute the drivers. - Simplify large data transfer from drivers to processor by obsoleting rpc. - Drivers don't need to create the complete list by taking advantage of python generators in order to conserve memory. - Lowering the total signiture of vitrage processes. - LockByDriver will enforce a driver does not run get_changes and get_all in parallel. Story: 2004384 Change-Id: Ie713456b2df96e24d0b15d2362a666162bfb4300
openstack · Nov 21, 2018 · 842f9d6 · 842f9d6
1 parent a4c466a
commit 842f9d6
Show file tree

Hide file tree

Showing 22 changed files with 187 additions and 419 deletions.
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
@@ -300,33 +300,17 @@ function start_vitrage {
         fi
     fi
 
-    run_process vitrage-collector "$VITRAGE_BIN_DIR/vitrage-collector --config-file $VITRAGE_CONF"
     run_process vitrage-graph "$VITRAGE_BIN_DIR/vitrage-graph --config-file $VITRAGE_CONF"
     run_process vitrage-notifier "$VITRAGE_BIN_DIR/vitrage-notifier --config-file $VITRAGE_CONF"
     run_process vitrage-ml "$VITRAGE_BIN_DIR/vitrage-ml --config-file $VITRAGE_CONF"
     run_process vitrage-persistor "$VITRAGE_BIN_DIR/vitrage-persistor --config-file $VITRAGE_CONF"
     run_process vitrage-snmp-parsing "$VITRAGE_BIN_DIR/vitrage-snmp-parsing --config-file $VITRAGE_CONF"
 
-    write_systemd_dependency vitrage-graph vitrage-collector
 
     change_systemd_kill_mode vitrage-graph
-    change_systemd_kill_mode vitrage-collector
 }
 
 
-function write_systemd_dependency {
-  local service_after=$1
-  local service_before=$2
-  local systemd_service_after="devstack@$service_after.service"
-  local systemd_service_before="devstack@$service_before.service"
-
-  local unitfile_after="$SYSTEMD_DIR/$systemd_service_after"
-
-  iniset -sudo $unitfile_after "Unit" "After" "$systemd_service_before"
-
-  $SYSTEMCTL daemon-reload
-}
-
 function change_systemd_kill_mode {
    local service=$1
    local systemd_service="devstack@$service.service"
@@ -343,7 +327,7 @@ function stop_vitrage {
         disable_apache_site vitrage
         restart_apache_server
     fi
-    for serv in vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-persistor vitrage-ml vitrage-snmp-parsing; do
+    for serv in vitrage-api vitrage-graph vitrage-notifier vitrage-persistor vitrage-ml vitrage-snmp-parsing; do
         stop_process $serv
     done
 }

diff --git a/devstack/settings b/devstack/settings
@@ -5,8 +5,6 @@ enable_service vitrage-api
 enable_service vitrage-graph
 # Notifier
 enable_service vitrage-notifier
-# Collector
-enable_service vitrage-collector
 # machine_learning
 enable_service vitrage-ml
 # Persistor

diff --git a/devstack/upgrade/settings b/devstack/upgrade/settings
@@ -7,7 +7,7 @@ VITRAGE_BASE_DEVSTACK_DIR=$TOP_DIR/../../old/vitrage/devstack
 VITRAGE_DIR=$TOP_DIR/../../new/vitrage
 source $VITRAGE_BASE_DEVSTACK_DIR/settings
 
-devstack_localrc base enable_service vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
-devstack_localrc target enable_service vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+devstack_localrc base enable_service vitrage-api vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+devstack_localrc target enable_service vitrage-api vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
 BASE_RUN_SMOKE=False
 TARGET_RUN_SMOKE=False
diff --git a/devstack/upgrade/shutdhown.sh b/devstack/upgrade/shutdhown.sh
@@ -29,7 +29,7 @@ set -o xtrace
 
 stop_vitrage
 
-SERVICES_DOWN="vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing"
+SERVICES_DOWN="vitrage-api vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing"
 
 # sanity check that services are actually down
 ensure_services_stopped $SERVICES_DOWN
diff --git a/devstack/upgrade/upgrade.sh b/devstack/upgrade/upgrade.sh
@@ -84,7 +84,7 @@ start_vitrage
 
 # Don't succeed unless the services come up
 # Truncating some service names to 11 characters
-ensure_services_started vitrage-api vitrage-collector vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
+ensure_services_started vitrage-api vitrage-graph vitrage-notifier vitrage-ml vitrage-persistor vitrage-snmp-parsing
 
 set +o xtrace
 echo "*********************************************************************"

diff --git a/doc/source/contributor/high-scale.rst b/doc/source/contributor/high-scale.rst
@@ -7,7 +7,8 @@ In a production environment with > 50,000 entities, the following configuration
 Tune RPC
 --------
 
-Vitrage-graph uses RPC to request data from vitrage-collector, these requests take longer, and there is a need to increase the timeout.
+Vitrage-api uses RPC to request data from vitrage-graph, these requests take longer, and there may be a need to
+increase the timeout.
 The following should be set in ``/etc/vitrage/vitrage.conf``, under ``[DEFAULT]`` section:
 
 +----------------------+---------------------------------------------------------+-----------------+-----------------+
@@ -20,8 +21,6 @@ To apply, restart these:
 
 ``sudo service vitrage-graph restart``
 
-``sudo service vitrage-collector restart``
-
 Restart the Vitrage api (either vitrage-api or apache)
 
 

diff --git a/doc/source/install/get_started.rst b/doc/source/install/get_started.rst
@@ -8,13 +8,12 @@ The Root Cause Analysis service consists of the following components:
 
 ``vitrage-graph`` service
   The main process. It includes the in-memory entity graph and the template evaluator.
+  Also responsible for retrieving data from the different datasources
 ``vitrage-notifier`` service
   Used for notifying external systems about Vitrage alarms/state changes. It only calls Nova force-down API
   and Simple Network Management Protocol (SNMP) in the Ocata release.
 ``vitrage-api`` service
   The API layer for Vitrage.
-``vitrage-collector`` service
-  Responsible for retrieving data from the different datasources.
 ``vitrage-ml`` service
   Performs alarm analysis using Machine Learning methods.
 ``vitrage-persistor`` service

diff --git a/doc/source/install/install-rdo.rst b/doc/source/install/install-rdo.rst
@@ -174,7 +174,6 @@ Run the following commands:
 
 .. code:: bash
 
-    vitrage-collector
     vitrage-graph
     vitrage-api
     vitrage-notifier

diff --git a/releasenotes/notes/collector-removal-fd805c6298d66eb0.yaml b/releasenotes/notes/collector-removal-fd805c6298d66eb0.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - Collector service removal to simplify and enhance scale performance.
+    vitrage-collector service was removed and vitrage-graph is responsible
+    to execute the drivers.
+    Allowing drivers to take advantage of python yield generators and conserve
+    memory.
diff --git a/setup.cfg b/setup.cfg
@@ -28,7 +28,6 @@ console_scripts =
     vitrage-api = vitrage.cli.api:main
     vitrage-graph = vitrage.cli.graph:main
     vitrage-notifier = vitrage.cli.notifier:main
-    vitrage-collector = vitrage.cli.collector:main
     vitrage-persistor = vitrage.cli.persistor:main
     vitrage-ml = vitrage.cli.machine_learning:main
     vitrage-dbsync = vitrage.cli.storage:dbsync

diff --git a/vitrage/cli/collector.py b/vitrage/cli/collector.py
diff --git a/vitrage/datasources/__init__.py b/vitrage/datasources/__init__.py
@@ -53,8 +53,4 @@
     cfg.StrOpt('notification_exchange',
                required=False,
                help='Exchange that is used for notifications.'),
-    cfg.StrOpt('notification_topic_collector',
-               default='vitrage_collector_notifications',
-               help='The topic on which event will be sent from the '
-                    'datasources to the graph processor')
 ]
diff --git a/vitrage/datasources/listener_service.py b/vitrage/datasources/listener_service.py
diff --git a/vitrage/datasources/rpc_service.py b/vitrage/datasources/rpc_service.py