From 226c2b23ecb92dfab36d1cede7fb87a69bf297df Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Thu, 13 Apr 2023 12:24:58 -0700 Subject: [PATCH 01/33] implemented neighboring agent discovery and agent to agent communication (#114) * implemented neighboring agent discovery and agent to agent communication * resolved issues on the pull request * added field topic for centralized services * Update to use cimgraph Bump Versions * updated to use cimgraph rather than cimlab * update to latest cimgraph --------- Co-authored-by: C <3979063+craig8@users.noreply.github.com> --- gridappsd/field_interface/__init__.py | 4 +- gridappsd/field_interface/agents/agents.py | 268 ++++++++++++------ gridappsd/field_interface/context.py | 56 ++-- .../field_interface/gridappsd_field_bus.py | 21 +- gridappsd/field_interface/interfaces.py | 191 +------------ gridappsd/goss.py | 2 +- gridappsd/topics.py | 88 +++++- pyproject.toml | 6 +- 8 files changed, 333 insertions(+), 303 deletions(-) diff --git a/gridappsd/field_interface/__init__.py b/gridappsd/field_interface/__init__.py index 2181da7..66eba6b 100644 --- a/gridappsd/field_interface/__init__.py +++ b/gridappsd/field_interface/__init__.py @@ -1,9 +1,9 @@ from typing import List -from gridappsd.field_interface.context import ContextManager +from gridappsd.field_interface.context import LocalContext from gridappsd.field_interface.interfaces import MessageBusDefinition __all__: List[str] = [ - "ContextManager", + "LocalContext", "MessageBusDefinition" ] diff --git a/gridappsd/field_interface/agents/agents.py b/gridappsd/field_interface/agents/agents.py index 6436733..68ae39b 100644 --- a/gridappsd/field_interface/agents/agents.py +++ b/gridappsd/field_interface/agents/agents.py @@ -1,46 +1,49 @@ -from typing import Dict -import cimlab.data_profile.cimext_2022 as cim - -from abc import abstractmethod -from dataclasses import dataclass, field +import dataclasses import importlib +import json import logging +from dataclasses import dataclass, field +from datetime import datetime +from typing import Dict -from gridappsd.field_interface.context import ContextManager - -from cimlab.loaders import Parameter, ConnectionParameters -from cimlab.loaders.gridappsd import GridappsdConnection, get_topology_response -from cimlab.models import SwitchArea, SecondaryArea, DistributedModel +from cimgraph.loaders import ConnectionParameters, gridappsd +from cimgraph.loaders.gridappsd import GridappsdConnection +from cimgraph.models import DistributedModel, SecondaryArea, SwitchArea +import gridappsd.topics as t +from gridappsd.field_interface.context import LocalContext from gridappsd.field_interface.gridappsd_field_bus import GridAPPSDMessageBus -from gridappsd.field_interface.interfaces import MessageBusDefinition - -import cimlab.data_profile.cimext_2022 as cim -from cimlab.loaders import Parameter, ConnectionParameters -from cimlab.loaders import gridappsd -from cimlab.loaders.gridappsd import GridappsdConnection, get_topology_response -from cimlab.models import SwitchArea, SecondaryArea, DistributedModel - +from gridappsd.field_interface.interfaces import (FieldMessageBus, + MessageBusDefinition) cim = None sparql = None - _log = logging.getLogger(__name__) + def set_cim_profile(cim_profile): global cim - cim = importlib.import_module('cimlab.data_profile.' + cim_profile) + cim = importlib.import_module('cimgraph.data_profile.' + cim_profile) gridappsd.set_cim_profile(cim_profile) - -class DistributedAgent: +@dataclass +class AgentRegistrationDetails: + agent_id: str + app_id: str + description: str + upstream_message_bus_id: FieldMessageBus.id + downstream_message_bus_id: FieldMessageBus.id + + +class DistributedAgent: def __init__(self, upstream_message_bus_def: MessageBusDefinition, downstream_message_bus_def: MessageBusDefinition, - agent_dict=None, + agent_config, + agent_area_dict=None, simulation_id=None, cim_profile: str = None): """ @@ -52,18 +55,29 @@ def __init__(self, self.downstream_message_bus = None self.simulation_id = simulation_id self.context = None + + #TODO: Change params and connection to local connection self.params = ConnectionParameters() self.connection = GridappsdConnection(self.params) + self.app_id = agent_config['app_id'] + self.description = agent_config['description'] + dt = datetime.now() + ts = datetime.timestamp(dt) + self.agent_id = "da_" + self.app_id + "_" + str(int(ts)) + self.agent_area_dict = agent_area_dict + if upstream_message_bus_def is not None: if upstream_message_bus_def.is_ot_bus: - self.upstream_message_bus = GridAPPSDMessageBus(upstream_message_bus_def) + self.upstream_message_bus = GridAPPSDMessageBus( + upstream_message_bus_def) # else: # self.upstream_message_bus = VolttronMessageBus(upstream_message_bus_def) if downstream_message_bus_def is not None: if downstream_message_bus_def.is_ot_bus: - self.downstream_message_bus = GridAPPSDMessageBus(downstream_message_bus_def) + self.downstream_message_bus = GridAPPSDMessageBus( + downstream_message_bus_def) # else: # self.downstream_message_bus = VolttronMessageBus(downstream_message_bus_def) @@ -73,35 +87,123 @@ def __init__(self, # self.addressable_equipments = agent_dict['addressable_equipment'] # self.unaddressable_equipments = agent_dict['unaddressable_equipment'] - @classmethod - def from_feeder(cls, feeder_id, area_id): - context = ContextManager.get_context_by_feeder(feeder_id, area_id) - return cls(context.get('upstream_message_bus_def'), context.get('downstream_message_bus_def')) - def connect(self): + + if self.agent_area_dict is None: + context = LocalContext.get_context_by_message_bus( + self.downstream_message_bus) + self.agent_area_dict = context['data'] + if self.upstream_message_bus is not None: self.upstream_message_bus.connect() if self.downstream_message_bus is not None: self.downstream_message_bus.connect() if self.downstream_message_bus is None and self.upstream_message_bus is None: - raise ValueError("Either upstream or downstream bus must be specified!") + raise ValueError( + "Either upstream or downstream bus must be specified!") + self.subscribe_to_measurement() + self.subscribe_to_messages() + self.subscribe_to_requests() + + if ('context_manager' not in self.app_id): + LocalContext.register_agent(self.downstream_message_bus, + self.upstream_message_bus, self) def subscribe_to_measurement(self): if self.simulation_id is None: - self.downstream_message_bus.subscribe(f"fieldbus/{self.downstream_message_bus.id}", self.on_measurement) + self.downstream_message_bus.subscribe( + t.field_output_topic(self.downstream_message_bus.id), + self.on_measurement) else: - topic = f"/topic/goss.gridappsd.field.simulation.output.{self.simulation_id}.{self.downstream_message_bus.id}" - _log.debug(f"subscribing to sim_output on topic {topic}") + topic = t.field_output_topic(self.downstream_message_bus.id, + self.simulation_id) + _log.debug(f"subscribing to simulation output on topic {topic}") self.downstream_message_bus.subscribe(topic, self.on_simulation_output) + def subscribe_to_messages(self): + + self.downstream_message_bus.subscribe( + t.field_message_bus_topic(self.downstream_message_bus), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_topic(self.upstream_message_bus), + self.on_upstream_message) + + _log.debug( + f"Subscribing to messages on application topics: \n {t.field_message_bus_app_topic(self.downstream_message_bus.id, self.app_id)} \ + \n {t.field_message_bus_app_topic(self.upstream_message_bus.id, self.app_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_message_bus_app_topic(self.downstream_message_bus.id, + self.app_id), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_app_topic(self.upstream_message_bus.id, + self.app_id), + self.on_upstream_message) + + _log.debug( + f"Subscribing to message on agents topics: \n {t.field_message_bus_agent_topic(self.downstream_message_bus.id, self.agent_id)} \ + \n {t.field_message_bus_agent_topic(self.upstream_message_bus.id, self.agent_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_message_bus_agent_topic(self.downstream_message_bus.id, + self.agent_id), + self.on_downstream_message) + self.downstream_message_bus.subscribe( + t.field_message_bus_agent_topic(self.upstream_message_bus.id, + self.agent_id), + self.on_upstream_message) + + def subscribe_to_requests(self): + + _log.debug( + f"Subscribing to requests on agents queue: \n {t.field_agent_request_queue(self.downstream_message_bus.id, self.agent_id)} \ + \n {t.field_agent_request_queue(self.upstream_message_bus.id, self.agent_id)}" + ) + self.downstream_message_bus.subscribe( + t.field_agent_request_queue(self.downstream_message_bus.id, + self.agent_id), + self.on_request_from_downstream) + self.downstream_message_bus.subscribe( + t.field_agent_request_queue(self.upstream_message_bus.id, + self.agent_id), + self.on_request_from_uptream) + def on_measurement(self, headers: Dict, message) -> None: - raise NotImplementedError(f"{self.__class__.__name__} must be overriden in child class") + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") def on_simulation_output(self, headers, message): self.on_measurement(headers=headers, message=message) + def on_upstream_message(self, headers: Dict, message) -> None: + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def on_downstream_message(self, headers: Dict, message) -> None: + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def on_request_from_uptream(self, headers: Dict, message): + self.on_request(self.upstream_message_bus, headers, message) + + def on_request_from_downstream(self, headers: Dict, message): + self.on_request(self.downstream_message_bus, headers, message) + + def on_request(self, message_bus, headers: Dict, message): + raise NotImplementedError( + f"{self.__class__.__name__} must be overriden in child class") + + def get_registration_details(self): + details = AgentRegistrationDetails(str(self.agent_id), self.app_id, + self.description, + self.upstream_message_bus.id, + self.downstream_message_bus.id) + return dataclasses.asdict(details) + ''' TODO this has not been implemented yet, so we are commented them out for now. # not all agent would use this @@ -110,20 +212,6 @@ def on_control(self, control): command = control.get('command') self.control_device(device_id, command) - def publish_to_upstream_bus(self,output): - self.switch_message_bus.publish(self.output_topic, output) - - # could be and upstream or peer level agent - def publish_to_upstream_bus_agent(self,agent_id, output): - self.switch_message_bus.publish(self.topic.agent_id, output) - - def publish_to_downstream_bus(self,message): - self.secondary_message_bus.publish(self.topic, message) - - # downstream agent - def publish_to_downstream_bus_agent(self,agent_id, message): - self.secondary_message_bus.publish(self.topic.agent_id, message) - def control_device(self, device_id, command): device_topic = self.devices.get(device_id) self.secondary_message_bus.publish(device_topic, command)''' @@ -131,46 +219,57 @@ def control_device(self, device_id, command): class FeederAgent(DistributedAgent): - def __init__(self, upstream_message_bus_def: MessageBusDefinition, + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, downstream_message_bus_def: MessageBusDefinition, - feeder_dict=None, simulation_id=None): - super(FeederAgent, self).__init__(upstream_message_bus_def, - downstream_message_bus_def, - feeder_dict, simulation_id) - + agent_config: Dict, + feeder_dict=None, + simulation_id=None): + super(FeederAgent, + self).__init__(upstream_message_bus_def, + downstream_message_bus_def, agent_config, + feeder_dict, simulation_id) + if feeder_dict is not None: feeder = cim.Feeder(mRID=downstream_message_bus_def.id) - - self.feeder_area = DistributedModel(connection=self.connection, feeder=feeder, topology=feeder_dict) + self.feeder_area = DistributedModel(connection=self.connection, + feeder=feeder, + topology=feeder_dict) class SwitchAreaAgent(DistributedAgent): - def __init__(self, upstream_message_bus_def: MessageBusDefinition, + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, downstream_message_bus_def: MessageBusDefinition, - switch_area_dict=None, simulation_id=None): - - super().__init__(upstream_message_bus_def, - downstream_message_bus_def, - switch_area_dict, simulation_id) - + agent_config: Dict, + switch_area_dict=None, + simulation_id=None): + + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, switch_area_dict, simulation_id) + if switch_area_dict is not None: - self.switch_area = SwitchArea(downstream_message_bus_def.id, self.connection) + self.switch_area = SwitchArea(downstream_message_bus_def.id, + self.connection) self.switch_area.initialize_switch_area(switch_area_dict) - + class SecondaryAreaAgent(DistributedAgent): - def __init__(self, upstream_message_bus_def: MessageBusDefinition, + def __init__(self, + upstream_message_bus_def: MessageBusDefinition, downstream_message_bus_def: MessageBusDefinition, - secondary_area_dict=None, simulation_id=None): - - super().__init__(upstream_message_bus_def, - downstream_message_bus_def, - secondary_area_dict, simulation_id) + agent_config: Dict, + secondary_area_dict=None, + simulation_id=None): + + super().__init__(upstream_message_bus_def, downstream_message_bus_def, + agent_config, secondary_area_dict, simulation_id) if secondary_area_dict is not None: - self.secondary_area = SecondaryArea(downstream_message_bus_def.id, self.connection) + self.secondary_area = SecondaryArea(downstream_message_bus_def.id, + self.connection) self.secondary_area.initialize_secondary_area(secondary_area_dict) @@ -185,12 +284,19 @@ class CoordinatingAgent: upstream, peer , downstream and broadcast """ - def __init__(self, feeder_id, system_message_bus_def: MessageBusDefinition, simulation_id=None): + def __init__(self, + feeder_id, + system_message_bus_def: MessageBusDefinition, + simulation_id=None): self.feeder_id = feeder_id self.distributed_agents = [] self.system_message_bus = GridAPPSDMessageBus(system_message_bus_def) self.system_message_bus.connect() + + #This will change when we have multiple feeders per system + self.downstream_message_bus = self.system_message_bus + # self.context = ContextManager.getContextByFeeder(self.feeder_id) # print(self.context) # self.addressable_equipments = self.context['data']['addressable_equipment'] @@ -205,20 +311,6 @@ def spawn_distributed_agent(self, distributed_agent: DistributedAgent): ''' - def on_message_from_feeder_bus(self, message): - pass - - def subscribe_to_distribution_bus(self, topic): - #self.system_message_bus.subscribe("/topic/goss.gridappsd.field."+self.feeder_id, - self.on_message_from_feeder_bus) - self.system_message_bus.subscribe(topic, self.on_message_from_feeder_bus) - - def subscribe_to_feeder_bus(self, topic): - self.system_message_bus.subscribe(topic, self.on_message_from_feeder_bus) - - def on_measurement(self, measurements): - print(measurements) - def on_control(self, control): device_id = control.get('device') command = control.get('command') diff --git a/gridappsd/field_interface/context.py b/gridappsd/field_interface/context.py index ae1c882..d6fa6c9 100644 --- a/gridappsd/field_interface/context.py +++ b/gridappsd/field_interface/context.py @@ -1,36 +1,52 @@ -from gridappsd import GridAPPSD +from gridappsd.field_interface.interfaces import FieldMessageBus +import dataclasses +import gridappsd.topics as t +import json -request_field_queue_prefix = 'goss.gridappsd.process.request.field' -request_field_context_queue = request_field_queue_prefix + '.context' -class ContextManager: - +class LocalContext: + @classmethod - def get_context_by_feeder(cls, feeder_mrid, area_id=None): - gridappsd_obj = GridAPPSD() - - request = {'modelId': feeder_mrid, + def get_context_by_feeder(cls, downstream_message_bus: FieldMessageBus, feeder_mrid, area_id=None): + + request = {'request_type' : 'get_context', + 'modelId': feeder_mrid, 'areaId': area_id} - - response = gridappsd_obj.get_response(request_field_context_queue, request) + response = downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request, timeout=10) return response @classmethod - def get_context_by_message_bus(cls, downstream_message_bus_id): + def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): """ - return agents/devices based on upstream and/or downstream message bus as input - make message bus id a list - - based on filter return distributed agents for different applications as well - """ - gridappsd_obj = GridAPPSD() + return agents/devices based on downstream message bus as input - request = {'downstream_message_bus_id': downstream_message_bus_id, + """ + request = {'request_type' : 'get_context', + 'downstream_message_bus_id': downstream_message_bus.id, 'agents': True, 'devices': True} + return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) + + @classmethod + def register_agent(cls, downstream_message_bus: FieldMessageBus, upstream_message_bus: FieldMessageBus, agent): + """ + Sends the newly created distributed agent's info to OT bus - return gridappsd_obj.get_response(request_field_context_queue, request) + """ + request = {'request_type' : 'register_agent', + 'agent' : agent.get_registration_details()} + downstream_message_bus.send(t.context_request_queue(downstream_message_bus.id), request) + upstream_message_bus.send(t.context_request_queue(upstream_message_bus.id), request) + + @classmethod + def get_agents(cls, downstream_message_bus: FieldMessageBus): + """ + Sends the newly created distributed agent's info to OT bus + + """ + request = {'request_type' : 'get_agents'} + return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) # Provide context based on router (ip trace) or PKI # Maybe able to emulate/simulate diff --git a/gridappsd/field_interface/gridappsd_field_bus.py b/gridappsd/field_interface/gridappsd_field_bus.py index 7d93f58..8cb1fa5 100644 --- a/gridappsd/field_interface/gridappsd_field_bus.py +++ b/gridappsd/field_interface/gridappsd_field_bus.py @@ -1,9 +1,10 @@ -from gridappsd.field_interface.interfaces import MessageBusDefinition -from gridappsd.field_interface.interfaces import FeederMessageBus from gridappsd import GridAPPSD +from gridappsd.field_interface.interfaces import FieldMessageBus +from gridappsd.field_interface.interfaces import MessageBusDefinition +from typing import Any -class GridAPPSDMessageBus(FeederMessageBus): +class GridAPPSDMessageBus(FieldMessageBus): def __init__(self, definition: MessageBusDefinition): super(GridAPPSDMessageBus, self).__init__(definition) self._id = definition.id @@ -36,12 +37,20 @@ def subscribe(self, topic, callback): def unsubscribe(self, topic): pass - def publish(self, data, topic: str = None): + def send(self, topic: str, message: Any): """ Publish device specific data to the concrete message bus. """ - pass - + if self.gridappsd_obj is not None: + self.gridappsd_obj.send(topic, message) + + def get_response(self, topic, message, timeout=5): + """ + Sends a message on a specific concrete queue, waits and returns the response + """ + if self.gridappsd_obj is not None: + return self.gridappsd_obj.get_response(topic, message, timeout) + def disconnect(self): """ Disconnect the device from the concrete message bus. diff --git a/gridappsd/field_interface/interfaces.py b/gridappsd/field_interface/interfaces.py index 3b776a9..8e42e66 100644 --- a/gridappsd/field_interface/interfaces.py +++ b/gridappsd/field_interface/interfaces.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum +import gridappsd.topics as t import logging from os import PathLike from pathlib import Path @@ -164,194 +165,24 @@ def unsubscribe(self, topic): pass @abstractmethod - def publish(self, data, topic: str = None): + def send(self, topic, message): """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. - """ - pass - - -class SwitchAreaMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - - @abstractmethod - def connect(self): - """ - Connect to the concrete message bus that implements this interface. - """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): - """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. - """ - pass - - -class SecondaryMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - - @abstractmethod - def connect(self): - """ - Connect to the concrete message bus that implements this interface. - """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): - """ - Publish device specific data to the concrete message bus. - """ - pass - - @abstractmethod - def disconnect(self): - """ - Disconnect the device from the concrete message bus. + Publish device specific message to the concrete message bus. """ pass - -class FeederMessageBus: - def __init__(self, config: MessageBusDefinition): - self._devices = dict() - self._is_ot_bus = config.is_ot_bus - self._id = config.id - - @property - def id(self): - return self._id - - @property - def is_ot_bus(self): - return self._is_ot_bus - - def add_device(self, device: "DeviceFieldInterface"): - self._devices[device.id] = device - - def disconnect_device(self, id: str): - del self._devices[id] - - @abstractmethod - def query_devices(self) -> dict: - pass - - @abstractmethod - def is_connected(self) -> bool: - """ - Is this object connected to the message bus - """ - pass - @abstractmethod - def connect(self): + def get_response(self, topic, message, timeout): """ - Connect to the concrete message bus that implements this interface. + Sends a message on a specific queue, waits and returns the response """ - pass - - @abstractmethod - def subscribe(self, topic, callback): - pass - - @abstractmethod - def unsubscribe(self, topic): - pass - - @abstractmethod - def publish(self, data, topic: str = None): + + def get_agent_response(self, agent_id, message, timeout): """ - Publish device specific data to the concrete message bus. + Sends a message on a specific agent's request queue, waits and returns the response """ - pass + topic = "{}.request.{}.{}".format(t.BASE_FIELD_QUEUE,self.id, agent_id) + self.get_response(topic, message, timeout) @abstractmethod def disconnect(self): @@ -361,8 +192,6 @@ def disconnect(self): pass - - class MessageBusDefinitions: def __init__( self, diff --git a/gridappsd/goss.py b/gridappsd/goss.py index 6757e67..8dff23f 100644 --- a/gridappsd/goss.py +++ b/gridappsd/goss.py @@ -356,7 +356,7 @@ def __init__(self): self._thread.start() def run_callbacks(self): - _log.info("Starting thread queue") + _log.debug("Starting thread queue") while True: cb, hdrs, msg = self._queue_callerback.get() try: diff --git a/gridappsd/topics.py b/gridappsd/topics.py index 22f3722..87bf567 100644 --- a/gridappsd/topics.py +++ b/gridappsd/topics.py @@ -44,6 +44,10 @@ FNCS_BASE_OUTPUT_TOPIC = '/topic/goss.gridappsd.simulation.output' BASE_SIMULATION_TOPIC = '/topic/goss.gridappsd.simulation' BASE_SIMULATION_LOG_TOPIC = "/topic/goss.gridappsd.simulation.log" +BASE_FIELD_TOPIC = '/topic/goss.gridappsd.field' + +BASE_FIELD_QUEUE = 'goss.gridappsd.field' +REGISTER_AGENT_QUEUE = 'goss.gridappsd.field.register.agent' BLAZEGRAPH = "/queue/goss.gridappsd.process.request.data.powergridmodel" # https://gridappsd.readthedocs.io/en/latest/using_gridappsd/index.html#querying-logs @@ -69,7 +73,6 @@ REQUEST_APP_START = ".".join((PROCESS_PREFIX, "request.app.start")) BASE_APPLICATION_HEARTBEAT = ".".join((BASE_TOPIC_PREFIX, "heartbeat")) - def platform_log_topic(): """ Utility method for getting the platform.log base topic """ @@ -171,3 +174,86 @@ def simulation_log_topic(simulation_id): """https://gridappsd.readthedocs.io/en/latest/using_gridappsd/index.html#subscribing-to-logs """ return "{}.{}".format(BASE_SIMULATION_LOG_TOPIC, simulation_id) + +def field_message_bus_topic(message_bus_id:str, app_id: str=None, agent_id: str=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param app_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + + return f"{BASE_FIELD_TOPIC}.{message_bus_id}.{app_id}.{agent_id}" + + +def field_message_bus_app_topic(message_bus_id, app_id=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param app_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, app_id) + +def field_message_bus_agent_topic(message_bus_id, agent_id=None): + """ Utility method for getting the publish/subscribe topic for a specific message bus. + + :param message_bus_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.{}.{}".format(BASE_FIELD_TOPIC, message_bus_id, agent_id) + +def field_agent_request_queue(message_bus_id, agent_id): + """ Utility method for getting the request topic for a specific distributed agent + + :param message_bus_id: + :param agent_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, agent_id) + +def context_request_queue(message_bus_id): + """ Utility method for getting the request topic for context manager + + :param message_bus_id: + :return: + """ + assert message_bus_id, "message_bus_id cannot be empty" + + return "{}.request.{}.{}".format(BASE_FIELD_QUEUE, message_bus_id, message_bus_id+'.context_manager') + +def field_output_topic(message_bus_id=None, simulation_id=None): + """ Utility method for getting the field output topic. + If message_bus_id is None, it returns topic used by centralized device interfaces to publish measurements. + If message_bus_id is not None, it returns topic used by distributed devices interfaces to publish measurements which is then subscribed by distributed agents. + + :param message_bus_id: + :param simulation_id + :return: str: Topic to receive field measurements + """ + + if simulation_id is None: + return "{}.{}".format(BASE_FIELD_TOPIC, "output") + else: + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.output",simulation_id,message_bus_id) + +def field_input_topic(message_bus_id=None, simulation_id=None): + """ Utility method for getting the field input topic. + If message_bus_id is None, it returns topic used by centralized device interfaces to subscribe to control commands. + If message_bus_id is not None, it returns topic used by distributed devices interfaces to subscribe to control commands. + + :param message_bus_id: + :param simulation_id + :return: str: Topic to receive input control commands + """ + + if simulation_id is None: + return "{}.{}".format(BASE_FIELD_TOPIC, "input") + else: + return "{}.{}.{}.{}".format(BASE_FIELD_TOPIC,"simulation.input",simulation_id,message_bus_id) diff --git a/pyproject.toml b/pyproject.toml index 0090974..ffa31bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python" -version = "v2.7.230209" +version = "v2.8.230413" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -33,10 +33,8 @@ python = ">=3.7.9,<4.0" PyYAML = "^6.0" pytz = "^2022.7" dateutils = "^0.6.7" -#gridappsd-cim-profile={url="https://github.com/GRIDAPPSD/gridappsd-cim-profile/releases/download/v0.7.20230120180831a0/gridappsd_cim_profile-0.7.20230120180831a0-py3-none-any.whl"} -#gridappsd-cim-profile = "^0.10.20230208223046a0" stomp-py = "6.0.0" -gridappsd-cim-lab = "^0.11.230209" +cim-graph = "^0.1.20230413185916a0" [tool.poetry.group.dev.dependencies] pytest = "^6.2.2" From e5f2b959e700ce0d36708fb4f0156b8064c21277 Mon Sep 17 00:00:00 2001 From: poorva1209 Date: Fri, 14 Apr 2023 14:32:33 -0700 Subject: [PATCH 02/33] resolved issues after testing. calling get context after message bus connection. --- gridappsd/field_interface/agents/agents.py | 22 +++++++++++----------- gridappsd/field_interface/context.py | 5 ++--- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/gridappsd/field_interface/agents/agents.py b/gridappsd/field_interface/agents/agents.py index 68ae39b..0ae2f7c 100644 --- a/gridappsd/field_interface/agents/agents.py +++ b/gridappsd/field_interface/agents/agents.py @@ -89,11 +89,6 @@ def __init__(self, def connect(self): - if self.agent_area_dict is None: - context = LocalContext.get_context_by_message_bus( - self.downstream_message_bus) - self.agent_area_dict = context['data'] - if self.upstream_message_bus is not None: self.upstream_message_bus.connect() if self.downstream_message_bus is not None: @@ -101,6 +96,11 @@ def connect(self): if self.downstream_message_bus is None and self.upstream_message_bus is None: raise ValueError( "Either upstream or downstream bus must be specified!") + + if self.agent_area_dict is None: + context = LocalContext.get_context_by_message_bus( + self.downstream_message_bus) + self.agent_area_dict = context['data'] self.subscribe_to_measurement() self.subscribe_to_messages() @@ -230,11 +230,11 @@ def __init__(self, downstream_message_bus_def, agent_config, feeder_dict, simulation_id) - if feeder_dict is not None: + if self.agent_area_dict is not None: feeder = cim.Feeder(mRID=downstream_message_bus_def.id) self.feeder_area = DistributedModel(connection=self.connection, feeder=feeder, - topology=feeder_dict) + topology=self.agent_area_dict) class SwitchAreaAgent(DistributedAgent): @@ -249,10 +249,10 @@ def __init__(self, super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, switch_area_dict, simulation_id) - if switch_area_dict is not None: + if self.agent_area_dict is not None: self.switch_area = SwitchArea(downstream_message_bus_def.id, self.connection) - self.switch_area.initialize_switch_area(switch_area_dict) + self.switch_area.initialize_switch_area(self.agent_area_dict) class SecondaryAreaAgent(DistributedAgent): @@ -267,10 +267,10 @@ def __init__(self, super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, secondary_area_dict, simulation_id) - if secondary_area_dict is not None: + if self.agent_area_dict is not None: self.secondary_area = SecondaryArea(downstream_message_bus_def.id, self.connection) - self.secondary_area.initialize_secondary_area(secondary_area_dict) + self.secondary_area.initialize_secondary_area(self.agent_area_dict) class CoordinatingAgent: diff --git a/gridappsd/field_interface/context.py b/gridappsd/field_interface/context.py index d6fa6c9..35dec86 100644 --- a/gridappsd/field_interface/context.py +++ b/gridappsd/field_interface/context.py @@ -23,9 +23,8 @@ def get_context_by_message_bus(cls, downstream_message_bus: FieldMessageBus): """ request = {'request_type' : 'get_context', - 'downstream_message_bus_id': downstream_message_bus.id, - 'agents': True, - 'devices': True} + 'downstream_message_bus_id': downstream_message_bus.id + } return downstream_message_bus.get_response(t.context_request_queue(downstream_message_bus.id), request) @classmethod From 01546322dab3f86b958f039198ca1a9b9139fc5e Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 4 May 2023 12:35:23 -0700 Subject: [PATCH 03/33] Update to use the new version number --- .github/workflows/dev-release.yml | 199 +++++++++++++++++------------- 1 file changed, 113 insertions(+), 86 deletions(-) diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml index b46e965..f161266 100644 --- a/.github/workflows/dev-release.yml +++ b/.github/workflows/dev-release.yml @@ -13,110 +13,137 @@ defaults: env: LANG: en_US.utf-8 LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.8' RUNS_ON: ubuntu-latest jobs: - - bump_version: + find-version-and-release: runs-on: ubuntu-latest steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python ${{ env.PYTHON_VERSION }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: latest - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2.1.7 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: | - poetry install --no-interaction + - name: Getting next version + id: find-version + uses: craig8/versioner@v1 - #---------------------------------------------- - # bump version number for patch - #---------------------------------------------- - - name: Bump Version - run: | - - # $versionarr now holds the current version of the tag. - IFS='.' read -ra versionarr <<< $(poetry version --short) - - today=$(date +'%Y%m%d%H%M%S') - - new_version="${versionarr[0]}.${versionarr[1]}.$today" - poetry version $new_version - poetry version prerelease - - NEW_TAG="v$(poetry version --short)" - echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - -# while [[ ! $(git tag -l "$tag_in_question") = '' ]] -# do -# poetry version prerelease -# tag_in_question="v$(poetry version --short)" -# loop - - #--------------------------------------------------------------- - # create build artifacts to be included as part of release - #--------------------------------------------------------------- - name: Create build artifacts run: | poetry build -vvv - - uses: ncipollo/release-action@v1 + - name: Do github release + uses: ncipollo/release-action@v1 + if: ${{ github.repository_owner }} == "GRIDAPPSD" with: artifacts: "dist/*.gz,dist/*.whl" artifactErrorsFailBuild: true generateReleaseNotes: true commit: ${{ github.ref }} prerelease: true - tag: ${{ env.NEW_TAG }} + tag: "v${{ steps.find-version.outputs.new-version }}" token: ${{ secrets.GITHUB_TOKEN }} - + - name: Publish pre-release to pypi if: github.repository_owner == 'GRIDAPPSD' run: | poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish \ No newline at end of file + poetry publish + +# bump_version: +# runs-on: ubuntu-latest +# steps: +# - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." +# - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" +# - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + +# #---------------------------------------------- +# # check-out repo and set-up python +# #---------------------------------------------- +# - name: Checkout code +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 + +# - name: Set up Python ${{ env.PYTHON_VERSION }} +# id: setup-python +# uses: actions/setup-python@v2 +# with: +# python-version: ${{ env.PYTHON_VERSION }} + +# #---------------------------------------------- +# # ----- install & configure poetry ----- +# #---------------------------------------------- +# - name: Install Poetry +# uses: snok/install-poetry@v1 +# with: +# version: latest +# virtualenvs-create: true +# virtualenvs-in-project: true +# installer-parallel: true + +# #---------------------------------------------- +# # load cached venv if cache exists +# #---------------------------------------------- +# - name: Load cached venv +# id: cached-poetry-dependencies +# uses: actions/cache@v2.1.7 +# with: +# path: .venv +# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + +# #---------------------------------------------- +# # install dependencies if cache does not exist +# #---------------------------------------------- +# - name: Install dependencies +# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' +# run: poetry install --no-interaction --no-root + +# #---------------------------------------------- +# # install your root project, if required +# #---------------------------------------------- +# - name: Install library +# run: | +# poetry install --no-interaction + +# #---------------------------------------------- +# # bump version number for patch +# #---------------------------------------------- +# - name: Bump Version +# run: | + +# # $versionarr now holds the current version of the tag. +# IFS='.' read -ra versionarr <<< $(poetry version --short) + +# today=$(date +'%Y%m%d%H%M%S') + +# new_version="${versionarr[0]}.${versionarr[1]}.$today" +# poetry version $new_version +# poetry version prerelease + +# NEW_TAG="v$(poetry version --short)" +# echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + +# # while [[ ! $(git tag -l "$tag_in_question") = '' ]] +# # do +# # poetry version prerelease +# # tag_in_question="v$(poetry version --short)" +# # loop + +# #--------------------------------------------------------------- +# # create build artifacts to be included as part of release +# #--------------------------------------------------------------- +# - name: Create build artifacts +# run: | +# poetry build -vvv + +# - uses: ncipollo/release-action@v1 +# with: +# artifacts: "dist/*.gz,dist/*.whl" +# artifactErrorsFailBuild: true +# generateReleaseNotes: true +# commit: ${{ github.ref }} +# prerelease: true +# tag: ${{ env.NEW_TAG }} +# token: ${{ secrets.GITHUB_TOKEN }} + +# - name: Publish pre-release to pypi +# if: github.repository_owner == 'GRIDAPPSD' +# run: | +# poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} +# poetry publish \ No newline at end of file From 872a19aa0bf726b7ac2f95108781b5bb0d9afb69 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 4 May 2023 12:55:07 -0700 Subject: [PATCH 04/33] Use main instead of tag --- .github/workflows/dev-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml index f161266..3c9bd06 100644 --- a/.github/workflows/dev-release.yml +++ b/.github/workflows/dev-release.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Getting next version id: find-version - uses: craig8/versioner@v1 + uses: craig8/versioner@main - name: Create build artifacts run: | @@ -146,4 +146,4 @@ jobs: # if: github.repository_owner == 'GRIDAPPSD' # run: | # poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} -# poetry publish \ No newline at end of file +# poetry publish From a63474037e2cb49edcc9d5e3b1c1605d0c2f1453 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 10 May 2023 11:29:41 -0700 Subject: [PATCH 05/33] Cleanse workflows --- .github/workflows/deploy-dev-release.yml | 61 ++++++++++ .github/workflows/dev-release.yml | 149 ----------------------- .github/workflows/dispatch-pypi.yml | 39 ------ .github/workflows/release_wheel.yml | 21 ---- 4 files changed, 61 insertions(+), 209 deletions(-) create mode 100644 .github/workflows/deploy-dev-release.yml delete mode 100644 .github/workflows/dev-release.yml delete mode 100644 .github/workflows/dispatch-pypi.yml delete mode 100644 .github/workflows/release_wheel.yml diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml new file mode 100644 index 0000000..764b423 --- /dev/null +++ b/.github/workflows/deploy-dev-release.yml @@ -0,0 +1,61 @@ +--- +name: Deploy Pre-Release Artifacts + +on: + workflow_dispatch: + inputs: + merge-strategy: + description: 'Merge strategy and strategy options. Used only in case of merge conflicts' + required: false + default: '' + type: string + release-version: + description: 'Version number to use. If provided bump-rule will be ignored' + required: false + default: '' + type: string + bump-rule: + description: 'Bump rule for computing next release version number.' + required: false + default: 'prerelease' + type: choice + options: + - patch + - minor + - major + - prepatch + - preminor + - premajor + - prerelease + run-tests-wait: + description: 'Wait time to run test after merge to main' + required: false + default: 600 + type: number + publish-to-test-pypi: + description: 'Set to true if you want to publish to https://test.pypi.org/legacy/ instead of pypi.org' + required: false + default: false + type: boolean + + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: '3.10' + +jobs: + call-deploy-release: + permissions: + contents: write # To push a branch + pull-requests: write # To create a PR from that branch + + uses: GRIDAPPSD/.github/.github/workflows/deploy-dev-release.yml@main + secrets: + git-token: ${{ secrets.GITHUB_TOKEN }} + pypi-token: ${{ secrets.PYPI_TOKEN }} + diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml deleted file mode 100644 index 3c9bd06..0000000 --- a/.github/workflows/dev-release.yml +++ /dev/null @@ -1,149 +0,0 @@ ---- -name: Deploy Pre-Release Artifacts - -on: - push: - branches: - - develop - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - RUNS_ON: ubuntu-latest - -jobs: - find-version-and-release: - runs-on: ubuntu-latest - steps: - - name: Getting next version - id: find-version - uses: craig8/versioner@main - - - name: Create build artifacts - run: | - poetry build -vvv - - - name: Do github release - uses: ncipollo/release-action@v1 - if: ${{ github.repository_owner }} == "GRIDAPPSD" - with: - artifacts: "dist/*.gz,dist/*.whl" - artifactErrorsFailBuild: true - generateReleaseNotes: true - commit: ${{ github.ref }} - prerelease: true - tag: "v${{ steps.find-version.outputs.new-version }}" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish pre-release to pypi - if: github.repository_owner == 'GRIDAPPSD' - run: | - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - poetry publish - -# bump_version: -# runs-on: ubuntu-latest -# steps: -# - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." -# - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" -# - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - -# #---------------------------------------------- -# # check-out repo and set-up python -# #---------------------------------------------- -# - name: Checkout code -# uses: actions/checkout@v2 -# with: -# fetch-depth: 0 - -# - name: Set up Python ${{ env.PYTHON_VERSION }} -# id: setup-python -# uses: actions/setup-python@v2 -# with: -# python-version: ${{ env.PYTHON_VERSION }} - -# #---------------------------------------------- -# # ----- install & configure poetry ----- -# #---------------------------------------------- -# - name: Install Poetry -# uses: snok/install-poetry@v1 -# with: -# version: latest -# virtualenvs-create: true -# virtualenvs-in-project: true -# installer-parallel: true - -# #---------------------------------------------- -# # load cached venv if cache exists -# #---------------------------------------------- -# - name: Load cached venv -# id: cached-poetry-dependencies -# uses: actions/cache@v2.1.7 -# with: -# path: .venv -# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - -# #---------------------------------------------- -# # install dependencies if cache does not exist -# #---------------------------------------------- -# - name: Install dependencies -# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' -# run: poetry install --no-interaction --no-root - -# #---------------------------------------------- -# # install your root project, if required -# #---------------------------------------------- -# - name: Install library -# run: | -# poetry install --no-interaction - -# #---------------------------------------------- -# # bump version number for patch -# #---------------------------------------------- -# - name: Bump Version -# run: | - -# # $versionarr now holds the current version of the tag. -# IFS='.' read -ra versionarr <<< $(poetry version --short) - -# today=$(date +'%Y%m%d%H%M%S') - -# new_version="${versionarr[0]}.${versionarr[1]}.$today" -# poetry version $new_version -# poetry version prerelease - -# NEW_TAG="v$(poetry version --short)" -# echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - -# # while [[ ! $(git tag -l "$tag_in_question") = '' ]] -# # do -# # poetry version prerelease -# # tag_in_question="v$(poetry version --short)" -# # loop - -# #--------------------------------------------------------------- -# # create build artifacts to be included as part of release -# #--------------------------------------------------------------- -# - name: Create build artifacts -# run: | -# poetry build -vvv - -# - uses: ncipollo/release-action@v1 -# with: -# artifacts: "dist/*.gz,dist/*.whl" -# artifactErrorsFailBuild: true -# generateReleaseNotes: true -# commit: ${{ github.ref }} -# prerelease: true -# tag: ${{ env.NEW_TAG }} -# token: ${{ secrets.GITHUB_TOKEN }} - -# - name: Publish pre-release to pypi -# if: github.repository_owner == 'GRIDAPPSD' -# run: | -# poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} -# poetry publish diff --git a/.github/workflows/dispatch-pypi.yml b/.github/workflows/dispatch-pypi.yml deleted file mode 100644 index c0d3209..0000000 --- a/.github/workflows/dispatch-pypi.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Dispatch to PyPi - -on: - workflow_dispatch: - -defaults: - run: - shell: bash - -env: - LANG: en_US.utf-8 - LC_ALL: en_US.utf-8 - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - -jobs: - - publish_to_pypi: - - runs-on: ubuntu-latest - - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" \ No newline at end of file diff --git a/.github/workflows/release_wheel.yml b/.github/workflows/release_wheel.yml deleted file mode 100644 index c6ac80a..0000000 --- a/.github/workflows/release_wheel.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Documentation located -# https://github.com/marketplace/actions/publish-python-poetry-package -name: Publish Python package -on: - push: - tags: - - 'v*.*.*' -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build and publish to pypi - uses: JRubics/poetry-publish@v1.7 - with: - # These are only needed when using test.pypi - #repository_name: testpypi - #repository_url: https://test.pypi.org/legacy/ - pypi_token: ${{ secrets.PYPI_TOKEN }} - ignore_dev_requirements: "yes" \ No newline at end of file From 4f7d1bfbf99a4c410aba1986d25dbd10fed4c378 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 10 May 2023 11:39:42 -0700 Subject: [PATCH 06/33] Update deploy-dev-release.yml --- .github/workflows/deploy-dev-release.yml | 40 +++--------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 764b423..9d1a448 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -2,43 +2,11 @@ name: Deploy Pre-Release Artifacts on: + push: + branches: + - develop workflow_dispatch: - inputs: - merge-strategy: - description: 'Merge strategy and strategy options. Used only in case of merge conflicts' - required: false - default: '' - type: string - release-version: - description: 'Version number to use. If provided bump-rule will be ignored' - required: false - default: '' - type: string - bump-rule: - description: 'Bump rule for computing next release version number.' - required: false - default: 'prerelease' - type: choice - options: - - patch - - minor - - major - - prepatch - - preminor - - premajor - - prerelease - run-tests-wait: - description: 'Wait time to run test after merge to main' - required: false - default: 600 - type: number - publish-to-test-pypi: - description: 'Set to true if you want to publish to https://test.pypi.org/legacy/ instead of pypi.org' - required: false - default: false - type: boolean - - + defaults: run: shell: bash From 5317a112bac6bdf726763587d1d6f4ec2c538a55 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 10 May 2023 11:46:42 -0700 Subject: [PATCH 07/33] cleanup --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7305a71..635e419 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,5 @@ def test_gridappsd_status(gridappsd_client): gappsd.set_service_status("Foo") assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - ``` From e6d7c40006cf142af621122b79a7cb13454971a1 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 15 May 2023 13:37:00 -0700 Subject: [PATCH 08/33] Initial monorepo structure --- .gitattributes | 1 - .pre-commit-config.yaml | 30 ++++++++++++ .../gridappsd}/field_interface/__init__.py | 0 .../field_interface/agents/__init__.py | 0 .../field_interface/agents/agents.py | 0 .../gridappsd}/field_interface/context.py | 0 .../field_interface/gridappsd_field_bus.py | 0 .../gridappsd}/field_interface/interfaces.py | 0 gridappsd-field-bus/pyproject.toml | 45 ++++++++++++++++++ .../conf/entrypoint.sh | 0 .../conf/run-gridappsd.sh | 0 .../gridappsd}/__init__.py | 0 .../gridappsd}/__main__.py | 0 .../gridappsd}/app_registration.py | 0 .../gridappsd}/difference_builder.py | 0 .../gridappsd}/docker_handler.py | 0 .../gridappsd}/goss.py | 0 .../gridappsd}/gridappsd.py | 0 .../gridappsd}/houses.py | 0 .../gridappsd}/loghandler.py | 0 .../gridappsd}/register_app.py | 0 .../gridappsd}/simulation.py | 0 .../gridappsd}/timeseries.py | 0 .../gridappsd}/topics.py | 0 .../gridappsd}/utils.py | 0 gridappsd-python/pyproject.toml | 47 +++++++++++++++++++ {tests => gridappsd-python/tests}/conftest.py | 0 .../13_node_2_min_base.json | 0 .../tests}/test_containers.py | 0 .../tests}/test_docker_handler.py | 0 .../tests}/test_goss.py | 0 .../tests}/test_gridappsd.py | 0 .../tests}/test_houses.py | 0 .../tests}/test_logging.py | 0 .../tests}/test_logging_integration.py | 0 .../tests}/test_simulation.py | 0 .../tests}/unittest_test_template.py | 0 poetry.toml | 2 + 38 files changed, 124 insertions(+), 1 deletion(-) delete mode 100644 .gitattributes create mode 100644 .pre-commit-config.yaml rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/__init__.py (100%) rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/agents/__init__.py (100%) rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/agents/agents.py (100%) rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/context.py (100%) rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/gridappsd_field_bus.py (100%) rename {gridappsd => gridappsd-field-bus/gridappsd}/field_interface/interfaces.py (100%) create mode 100644 gridappsd-field-bus/pyproject.toml rename {gridappsd => gridappsd-python}/conf/entrypoint.sh (100%) rename {gridappsd => gridappsd-python}/conf/run-gridappsd.sh (100%) rename {gridappsd => gridappsd-python/gridappsd}/__init__.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/__main__.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/app_registration.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/difference_builder.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/docker_handler.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/goss.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/gridappsd.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/houses.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/loghandler.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/register_app.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/simulation.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/timeseries.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/topics.py (100%) rename {gridappsd => gridappsd-python/gridappsd}/utils.py (100%) create mode 100644 gridappsd-python/pyproject.toml rename {tests => gridappsd-python/tests}/conftest.py (100%) rename {tests => gridappsd-python/tests}/simulation_fixtures/13_node_2_min_base.json (100%) rename {tests => gridappsd-python/tests}/test_containers.py (100%) rename {tests => gridappsd-python/tests}/test_docker_handler.py (100%) rename {tests => gridappsd-python/tests}/test_goss.py (100%) rename {tests => gridappsd-python/tests}/test_gridappsd.py (100%) rename {tests => gridappsd-python/tests}/test_houses.py (100%) rename {tests => gridappsd-python/tests}/test_logging.py (100%) rename {tests => gridappsd-python/tests}/test_logging_integration.py (100%) rename {tests => gridappsd-python/tests}/test_simulation.py (100%) rename {tests => gridappsd-python/tests}/unittest_test_template.py (100%) create mode 100644 poetry.toml diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 4ad0ecf..0000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -tests/measurment-13-node-120s.json filter=lfs diff=lfs merge=lfs -text diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..94c3e96 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-xml + - id: forbid-new-submodules + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-merge-conflict + - id: no-commit-to-branch # blocks main commits. To bypass do git commit --allow-empty + - id: pretty-format-json + + +#- repo: https://github.com/pre-commit/mirrors-autopep8 +# rev: v1.6.0 +# hooks: +# - id: autopep8 + +- repo: https://github.com/craig8/mirrors-yapf + rev: b84f670025671a341d0afd2b06b877b195d65c0f # Use the sha / tag you want to point at + hooks: + - id: yapf + name: yapf + description: "A formatter for Python files." + entry: yapf + language: python + types: [ python ] \ No newline at end of file diff --git a/gridappsd/field_interface/__init__.py b/gridappsd-field-bus/gridappsd/field_interface/__init__.py similarity index 100% rename from gridappsd/field_interface/__init__.py rename to gridappsd-field-bus/gridappsd/field_interface/__init__.py diff --git a/gridappsd/field_interface/agents/__init__.py b/gridappsd-field-bus/gridappsd/field_interface/agents/__init__.py similarity index 100% rename from gridappsd/field_interface/agents/__init__.py rename to gridappsd-field-bus/gridappsd/field_interface/agents/__init__.py diff --git a/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus/gridappsd/field_interface/agents/agents.py similarity index 100% rename from gridappsd/field_interface/agents/agents.py rename to gridappsd-field-bus/gridappsd/field_interface/agents/agents.py diff --git a/gridappsd/field_interface/context.py b/gridappsd-field-bus/gridappsd/field_interface/context.py similarity index 100% rename from gridappsd/field_interface/context.py rename to gridappsd-field-bus/gridappsd/field_interface/context.py diff --git a/gridappsd/field_interface/gridappsd_field_bus.py b/gridappsd-field-bus/gridappsd/field_interface/gridappsd_field_bus.py similarity index 100% rename from gridappsd/field_interface/gridappsd_field_bus.py rename to gridappsd-field-bus/gridappsd/field_interface/gridappsd_field_bus.py diff --git a/gridappsd/field_interface/interfaces.py b/gridappsd-field-bus/gridappsd/field_interface/interfaces.py similarity index 100% rename from gridappsd/field_interface/interfaces.py rename to gridappsd-field-bus/gridappsd/field_interface/interfaces.py diff --git a/gridappsd-field-bus/pyproject.toml b/gridappsd-field-bus/pyproject.toml new file mode 100644 index 0000000..6d37b53 --- /dev/null +++ b/gridappsd-field-bus/pyproject.toml @@ -0,0 +1,45 @@ +[tool.poetry] +name = "gridappsd-field-bus" +version = "v2.8.230413" +description = "GridAPPS-D Field Bus Implementation" +authors = [ + "C. Allwardt <3979063+craig8@users.noreply.github.com>", + "P. Sharma =1.2.0"] +build-backend = "poetry.core.masonry.api" diff --git a/gridappsd/conf/entrypoint.sh b/gridappsd-python/conf/entrypoint.sh similarity index 100% rename from gridappsd/conf/entrypoint.sh rename to gridappsd-python/conf/entrypoint.sh diff --git a/gridappsd/conf/run-gridappsd.sh b/gridappsd-python/conf/run-gridappsd.sh similarity index 100% rename from gridappsd/conf/run-gridappsd.sh rename to gridappsd-python/conf/run-gridappsd.sh diff --git a/gridappsd/__init__.py b/gridappsd-python/gridappsd/__init__.py similarity index 100% rename from gridappsd/__init__.py rename to gridappsd-python/gridappsd/__init__.py diff --git a/gridappsd/__main__.py b/gridappsd-python/gridappsd/__main__.py similarity index 100% rename from gridappsd/__main__.py rename to gridappsd-python/gridappsd/__main__.py diff --git a/gridappsd/app_registration.py b/gridappsd-python/gridappsd/app_registration.py similarity index 100% rename from gridappsd/app_registration.py rename to gridappsd-python/gridappsd/app_registration.py diff --git a/gridappsd/difference_builder.py b/gridappsd-python/gridappsd/difference_builder.py similarity index 100% rename from gridappsd/difference_builder.py rename to gridappsd-python/gridappsd/difference_builder.py diff --git a/gridappsd/docker_handler.py b/gridappsd-python/gridappsd/docker_handler.py similarity index 100% rename from gridappsd/docker_handler.py rename to gridappsd-python/gridappsd/docker_handler.py diff --git a/gridappsd/goss.py b/gridappsd-python/gridappsd/goss.py similarity index 100% rename from gridappsd/goss.py rename to gridappsd-python/gridappsd/goss.py diff --git a/gridappsd/gridappsd.py b/gridappsd-python/gridappsd/gridappsd.py similarity index 100% rename from gridappsd/gridappsd.py rename to gridappsd-python/gridappsd/gridappsd.py diff --git a/gridappsd/houses.py b/gridappsd-python/gridappsd/houses.py similarity index 100% rename from gridappsd/houses.py rename to gridappsd-python/gridappsd/houses.py diff --git a/gridappsd/loghandler.py b/gridappsd-python/gridappsd/loghandler.py similarity index 100% rename from gridappsd/loghandler.py rename to gridappsd-python/gridappsd/loghandler.py diff --git a/gridappsd/register_app.py b/gridappsd-python/gridappsd/register_app.py similarity index 100% rename from gridappsd/register_app.py rename to gridappsd-python/gridappsd/register_app.py diff --git a/gridappsd/simulation.py b/gridappsd-python/gridappsd/simulation.py similarity index 100% rename from gridappsd/simulation.py rename to gridappsd-python/gridappsd/simulation.py diff --git a/gridappsd/timeseries.py b/gridappsd-python/gridappsd/timeseries.py similarity index 100% rename from gridappsd/timeseries.py rename to gridappsd-python/gridappsd/timeseries.py diff --git a/gridappsd/topics.py b/gridappsd-python/gridappsd/topics.py similarity index 100% rename from gridappsd/topics.py rename to gridappsd-python/gridappsd/topics.py diff --git a/gridappsd/utils.py b/gridappsd-python/gridappsd/utils.py similarity index 100% rename from gridappsd/utils.py rename to gridappsd-python/gridappsd/utils.py diff --git a/gridappsd-python/pyproject.toml b/gridappsd-python/pyproject.toml new file mode 100644 index 0000000..432b471 --- /dev/null +++ b/gridappsd-python/pyproject.toml @@ -0,0 +1,47 @@ +[tool.poetry] +name = "gridappsd-python" +version = "v2.8.230413" +description = "A GridAPPS-D Python Adapter" +authors = [ + "C. Allwardt <3979063+craig8@users.noreply.github.com>", + "P. Sharma =1.2.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tests/conftest.py b/gridappsd-python/tests/conftest.py similarity index 100% rename from tests/conftest.py rename to gridappsd-python/tests/conftest.py diff --git a/tests/simulation_fixtures/13_node_2_min_base.json b/gridappsd-python/tests/simulation_fixtures/13_node_2_min_base.json similarity index 100% rename from tests/simulation_fixtures/13_node_2_min_base.json rename to gridappsd-python/tests/simulation_fixtures/13_node_2_min_base.json diff --git a/tests/test_containers.py b/gridappsd-python/tests/test_containers.py similarity index 100% rename from tests/test_containers.py rename to gridappsd-python/tests/test_containers.py diff --git a/tests/test_docker_handler.py b/gridappsd-python/tests/test_docker_handler.py similarity index 100% rename from tests/test_docker_handler.py rename to gridappsd-python/tests/test_docker_handler.py diff --git a/tests/test_goss.py b/gridappsd-python/tests/test_goss.py similarity index 100% rename from tests/test_goss.py rename to gridappsd-python/tests/test_goss.py diff --git a/tests/test_gridappsd.py b/gridappsd-python/tests/test_gridappsd.py similarity index 100% rename from tests/test_gridappsd.py rename to gridappsd-python/tests/test_gridappsd.py diff --git a/tests/test_houses.py b/gridappsd-python/tests/test_houses.py similarity index 100% rename from tests/test_houses.py rename to gridappsd-python/tests/test_houses.py diff --git a/tests/test_logging.py b/gridappsd-python/tests/test_logging.py similarity index 100% rename from tests/test_logging.py rename to gridappsd-python/tests/test_logging.py diff --git a/tests/test_logging_integration.py b/gridappsd-python/tests/test_logging_integration.py similarity index 100% rename from tests/test_logging_integration.py rename to gridappsd-python/tests/test_logging_integration.py diff --git a/tests/test_simulation.py b/gridappsd-python/tests/test_simulation.py similarity index 100% rename from tests/test_simulation.py rename to gridappsd-python/tests/test_simulation.py diff --git a/tests/unittest_test_template.py b/gridappsd-python/tests/unittest_test_template.py similarity index 100% rename from tests/unittest_test_template.py rename to gridappsd-python/tests/unittest_test_template.py diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true From 1a5ede0817866f2fd2ac62f58f2218e2361f1100 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 15 May 2023 16:39:02 -0700 Subject: [PATCH 09/33] Moved to using lib in the subdir to distinguish from top level checkout --- .gitignore | 4 + CHANGELOG.md | 0 VERSION | 1 + gridappsd-field-bus-lib/CHANGELOG.md | 0 gridappsd-field-bus-lib/README.md | 0 .../gridappsd/field_interface/__init__.py | 0 .../field_interface/agents/__init__.py | 0 .../field_interface/agents/agents.py | 0 .../gridappsd/field_interface/context.py | 0 .../field_interface/gridappsd_field_bus.py | 0 .../gridappsd/field_interface/interfaces.py | 0 .../pyproject.toml | 2 +- gridappsd-python-lib/README.md | 328 ++++++++++++++++++ .../conf/entrypoint.sh | 0 .../conf/run-gridappsd.sh | 0 .../gridappsd/__init__.py | 0 .../gridappsd/__main__.py | 0 .../gridappsd/app_registration.py | 0 .../gridappsd/difference_builder.py | 0 .../gridappsd/docker_handler.py | 0 .../gridappsd/goss.py | 0 .../gridappsd/gridappsd.py | 0 .../gridappsd/houses.py | 0 .../gridappsd/loghandler.py | 0 .../gridappsd/register_app.py | 0 .../gridappsd/simulation.py | 0 .../gridappsd/timeseries.py | 0 .../gridappsd/topics.py | 0 .../gridappsd/utils.py | 0 .../pyproject.toml | 0 .../tests/conftest.py | 0 .../13_node_2_min_base.json | 0 .../tests/test_containers.py | 0 .../tests/test_docker_handler.py | 0 .../tests/test_goss.py | 0 .../tests/test_gridappsd.py | 0 .../tests/test_houses.py | 0 .../tests/test_logging.py | 0 .../tests/test_logging_integration.py | 0 .../tests/test_simulation.py | 0 .../tests/unittest_test_template.py | 0 pyproject.toml | 23 -- 42 files changed, 334 insertions(+), 24 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 VERSION create mode 100644 gridappsd-field-bus-lib/CHANGELOG.md create mode 100644 gridappsd-field-bus-lib/README.md rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/__init__.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/agents/__init__.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/agents/agents.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/context.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/gridappsd_field_bus.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/gridappsd/field_interface/interfaces.py (100%) rename {gridappsd-field-bus => gridappsd-field-bus-lib}/pyproject.toml (93%) create mode 100644 gridappsd-python-lib/README.md rename {gridappsd-python => gridappsd-python-lib}/conf/entrypoint.sh (100%) rename {gridappsd-python => gridappsd-python-lib}/conf/run-gridappsd.sh (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/__init__.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/__main__.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/app_registration.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/difference_builder.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/docker_handler.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/goss.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/gridappsd.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/houses.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/loghandler.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/register_app.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/simulation.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/timeseries.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/topics.py (100%) rename {gridappsd-python => gridappsd-python-lib}/gridappsd/utils.py (100%) rename {gridappsd-python => gridappsd-python-lib}/pyproject.toml (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/conftest.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/simulation_fixtures/13_node_2_min_base.json (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_containers.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_docker_handler.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_goss.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_gridappsd.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_houses.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_logging.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_logging_integration.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/test_simulation.py (100%) rename {gridappsd-python => gridappsd-python-lib}/tests/unittest_test_template.py (100%) diff --git a/.gitignore b/.gitignore index fda0577..751a919 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,7 @@ ENV/ # visual studio code .vscode/ + +# pytest cache +.pytest_cache + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-field-bus-lib/CHANGELOG.md b/gridappsd-field-bus-lib/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/README.md b/gridappsd-field-bus-lib/README.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus/gridappsd/field_interface/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/__init__.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/__init__.py diff --git a/gridappsd-field-bus/gridappsd/field_interface/agents/__init__.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/agents/__init__.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/agents/__init__.py diff --git a/gridappsd-field-bus/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/agents/agents.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py diff --git a/gridappsd-field-bus/gridappsd/field_interface/context.py b/gridappsd-field-bus-lib/gridappsd/field_interface/context.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/context.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/context.py diff --git a/gridappsd-field-bus/gridappsd/field_interface/gridappsd_field_bus.py b/gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/gridappsd_field_bus.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/gridappsd_field_bus.py diff --git a/gridappsd-field-bus/gridappsd/field_interface/interfaces.py b/gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py similarity index 100% rename from gridappsd-field-bus/gridappsd/field_interface/interfaces.py rename to gridappsd-field-bus-lib/gridappsd/field_interface/interfaces.py diff --git a/gridappsd-field-bus/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml similarity index 93% rename from gridappsd-field-bus/pyproject.toml rename to gridappsd-field-bus-lib/pyproject.toml index 6d37b53..c01cb53 100644 --- a/gridappsd-field-bus/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -30,7 +30,7 @@ gridappsd-cli = 'gridappsd.cli:_main' [tool.poetry.dependencies] python = ">=3.7.9,<4.0" -gridappsd-python = {path="../gridappsd-python", develop=true} +gridappsd-python = {path="../gridappsd-python-lib", develop=true} cim-graph = "^0.1.20230413185916a0" [tool.poetry.group.dev.dependencies] diff --git a/gridappsd-python-lib/README.md b/gridappsd-python-lib/README.md new file mode 100644 index 0000000..635e419 --- /dev/null +++ b/gridappsd-python-lib/README.md @@ -0,0 +1,328 @@ +[![Run All Pytests](https://github.com/GRIDAPPSD/gridappsd-python/actions/workflows/run-pytest.yml/badge.svg)](https://github.com/GRIDAPPSD/gridappsd-python/actions/workflows/run-pytest.yml) + +# gridappsd-python +Python library for developing applications and services against the gridappsd api + +## Requirements + +The gridappsd-python library requires a python version >= 3.6 and < 4 in order to work properly (Note no testing +has been done with python 4 to date). + +## Installation + +The recommended installation of `gridappsd-python` is in a separate virtual environment. Executing the following +will create an environment called `griddapps-env`. + +```shell +python3 -m venv gridappsd-env +``` + +Sourcing the gridappsd-env activates the newly created python environment. + +```shell +source gridappsd-env/bin/activate +``` + +Upgrade pip to the latest (some packages require 19.0+ version of pip). + +```shell +python -m pip install pip --upgrade +``` + +Install the latest `gridappsd-python` and its dependencies in the virtual environment. + +```shell +pip install gridappsd-python +``` + +### Verifying things are working properly + +The following code snippet assumes you have created a gridappsd instance using the steps in +https://github.com/GRIDAPPSD/gridappsd-docker. + +Create a test script (tester.py) with the following content. + +```python + +from gridappsd import GridAPPSD + +def on_message_callback(header, message): + print(f"header: {header} message: {message}") + +# Note these should be changed on the server in a cyber secure environment! +username = "app_user" +password = "1234App" + +# Note: there are other parameters for connecting to +# systems other than localhost +gapps = GridAPPSD(username=username, password=password) + +assert gapps.connected + +gapps.send('send.topic', {"foo": "bar"}) + +# Note we are sending the function not executing the function in the second parameter +gapps.subscribe('subscribe.topic', on_message_callback) + +gapps.send('subcribe.topic', 'A message about subscription') + +time.sleep(5) + +gapps.close() + +``` + +Start up the gridappsd-docker enabled platform. Then run the following to execute the tester.py script + +```shell +python tester.py +``` + +## Application Developers + +### Deployment + +Please see [DOCKER_CONTAINER.md](DOCKER_CONTAINER.md) for working within the docker application base container. + +### Local Development + +Developing applications against gridappsd using the `gridappsd-python` library should follow the same steps +as above, however with a couple of environmental variables specified. The following environmental variables are +available to provide the same context that would be available from inside the application docker container. These +are useful to know for developing your application outside of the docker context (e.g. in a python notebook). + +***NOTE: you can also define these your ~./bashrc file so you don't have to specify them all the time*** + +```shell +# export allows all processes started by this shell to have access to the global variable + +# address where the gridappsd server is running - default localhost +export GRIDAPPSD_ADDRESS=localhost + +# port to connect to on the gridappsd server (the stomp client port) +export GRIDAPPSD_PORT=61613 + +# username to connect to the gridappsd server +export GRIDAPPSD_USER=app_user + +# password to connect to the gridappsd server +export GRIDAPPSD_PASSWORD=1234App + +# Note these should be changed on the server in a cyber secure environment! +``` + +The following is the same tester code as above, but with the environment variables set. The environment variables +should be set in your environment when running the application inside our docker container. + +```python + +from gridappsd import GridAPPSD + +def on_message_callback(header, message): + print(f"header: {header} message: {message}") + +# Create GridAPPSD object and connect to the gridappsd server. +gapps = GridAPPSD() + +assert gapps.connected + +gapps.send('send.topic', {"foo": "bar"}) + +# Note we are sending the function not executing the function in the second parameter +gapps.subscribe('subscribe.topic', on_message_callback) + +gapps.send('subcribe.topic', 'A message about subscription') + +time.sleep(5) + +gapps.close() + +``` + +## Developers + +This project uses poetry to build the environment for execution. Follow the instructions +https://python-poetry.org/docs/#installation to install poetry. As a developer I prefer not to have poetry installed +in the same virtual environment that my projects are in. + +Clone the github repository: + +```shell +git clone https://github.com/GRIDAPPSD/gridappsd-python -b develop +cd gridappsd-python +``` + +The following commands build and install a local wheel into an environment created just for this package. + +```shell +# Build the project (stores in dist directory both .tar.gz and .whl file) +poetry build + +# Install the wheel into the environment and the dev dependencies +poetry install + +# Install only the library dependencies +poetry install --no-dev +``` + +***Note:*** Poetry does not have a setup.py that you can install in editable mode like with pip install -e ., however +you can extract the generated setup.py file from the built tar.gz file in the dist directory. Just extract the +.tar.gz file and copy the setup.py file from the extracted directory to the root of gridappsd-python. Then you can +enable editing through pip install -e. as normal. + + +## Testing + +Testing has become an integral part of the software lifecycle. The `gridappsd-python` library has both unit and +integration tests available to be run. In order to execute these, you must have installed the gridappsd-python library +as above with dev-dependencies. + +During the testing phase the docker containers required for the tests are downloaded from +dockerhub and started. By default the `develop` tag is used to test the library using pytest. +One can customize the docker image tag by setting the environmental +variable `GRIDAPPSD_TAG_ENV` either by `export GRIDAPPSD_TAG_ENV=other_tag` or by executing +pytest with the following: + +```shell script + +# Export environmental variables and all tests will use the same tag (other_tag) to pull from docker hub. +# Default tag is develop +export GRIDAPPSD_TAG_ENV=other_tag +pytest + +# Tests also require the username and password to be avaialable as environmental variables +# in order for them to properly run these tests +export GRIDAPPSD_USER=user +export GRIDAPPSD_PASSWORD=pass + +pytest +``` + + ***NOTE: the first running the tests will download all of the docker images associated with the + [GOSS-GridAPPS-D](http://github.com/GRIDAPPSD/GOSS-GridAPPS-D) repository. This process may take some time.*** + +### Running tests created in a new project + +The `gridappsd-python` library exposes a testing environment through the `gridappsd.docker_handler` module. Including the following +`conftest.py` in the root of your base test directory allows tests to reference these. Using these fixtures will start all of the +base containers required for `gridappsd` to run. + +```python + +# conftest.py +# Create a conftest.py file in the root of the tests directory to enable usage throughout the tests directory and below. +# +# Tested project structure an layout +# +# project-folder\ +# mainmodule\ +# __init__.py +# myapplication.py +# tests\ +# conftest.py +# test_myapplication.py +# README.md + +import logging +import os +import sys + +import pytest + +from gridappsd import GridAPPSD, GOSS +from gridappsd.docker_handler import run_dependency_containers, run_gridappsd_container, Containers + +levels = dict( + CRITICAL=50, + FATAL=50, + ERROR=40, + WARNING=30, + WARN=30, + INFO=20, + DEBUG=10, + NOTSET=0 +) + +# Get string representation of the log level passed +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") + +# Make sure the level passed is one of the valid levels. +if LOG_LEVEL not in levels.keys(): + raise AttributeError("Invalid LOG_LEVEL environmental variable set.") + +# Set the numeric version of log level to pass to the basicConfig function +LOG_LEVEL = levels[LOG_LEVEL] + +logging.basicConfig(stream=sys.stdout, level=LOG_LEVEL, + format="%(asctime)s|%(levelname)s|%(name)s|%(message)s") +logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +logging.getLogger("docker.utils.config").setLevel(logging.INFO) +logging.getLogger("docker.auth").setLevel(logging.INFO) + + +STOP_CONTAINER_AFTER_TEST = os.environ.get('GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', True) + + +@pytest.fixture(scope="module") +def docker_dependencies(): + print("Docker dependencies") + # Containers.reset_all_containers() + + with run_dependency_containers(stop_after=STOP_CONTAINER_AFTER_TEST) as dep: + yield dep + print("Cleanup docker dependencies") + + +@pytest.fixture +def gridappsd_client(request, docker_dependencies): + with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + gappsd = GridAPPSD() + gappsd.connect() + assert gappsd.connected + models = gappsd.query_model_names() + assert models is not None + if request.cls is not None: + request.cls.gridappsd_client = gappsd + yield gappsd + + gappsd.disconnect() + + +@pytest.fixture +def goss_client(docker_dependencies): + with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + goss = GOSS() + goss.connect() + assert goss.connected + + yield goss + +``` + +Using the above fixtures from inside a test module and test function looks like the following: + +```python + +# Example test function using the gridappsd_client fixture + +@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) +def test_gridappsd_status(gridappsd_client): + gappsd = gridappsd_client + assert "helics_goss_bridge.py" == gappsd.get_application_id() + assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value + assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value + gappsd.set_application_status("RUNNING") + + assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value + assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value + + gappsd.set_service_status("COMPLETE") + assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value + + # Invalid + gappsd.set_service_status("Foo") + assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value + assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value +``` + diff --git a/gridappsd-python/conf/entrypoint.sh b/gridappsd-python-lib/conf/entrypoint.sh similarity index 100% rename from gridappsd-python/conf/entrypoint.sh rename to gridappsd-python-lib/conf/entrypoint.sh diff --git a/gridappsd-python/conf/run-gridappsd.sh b/gridappsd-python-lib/conf/run-gridappsd.sh similarity index 100% rename from gridappsd-python/conf/run-gridappsd.sh rename to gridappsd-python-lib/conf/run-gridappsd.sh diff --git a/gridappsd-python/gridappsd/__init__.py b/gridappsd-python-lib/gridappsd/__init__.py similarity index 100% rename from gridappsd-python/gridappsd/__init__.py rename to gridappsd-python-lib/gridappsd/__init__.py diff --git a/gridappsd-python/gridappsd/__main__.py b/gridappsd-python-lib/gridappsd/__main__.py similarity index 100% rename from gridappsd-python/gridappsd/__main__.py rename to gridappsd-python-lib/gridappsd/__main__.py diff --git a/gridappsd-python/gridappsd/app_registration.py b/gridappsd-python-lib/gridappsd/app_registration.py similarity index 100% rename from gridappsd-python/gridappsd/app_registration.py rename to gridappsd-python-lib/gridappsd/app_registration.py diff --git a/gridappsd-python/gridappsd/difference_builder.py b/gridappsd-python-lib/gridappsd/difference_builder.py similarity index 100% rename from gridappsd-python/gridappsd/difference_builder.py rename to gridappsd-python-lib/gridappsd/difference_builder.py diff --git a/gridappsd-python/gridappsd/docker_handler.py b/gridappsd-python-lib/gridappsd/docker_handler.py similarity index 100% rename from gridappsd-python/gridappsd/docker_handler.py rename to gridappsd-python-lib/gridappsd/docker_handler.py diff --git a/gridappsd-python/gridappsd/goss.py b/gridappsd-python-lib/gridappsd/goss.py similarity index 100% rename from gridappsd-python/gridappsd/goss.py rename to gridappsd-python-lib/gridappsd/goss.py diff --git a/gridappsd-python/gridappsd/gridappsd.py b/gridappsd-python-lib/gridappsd/gridappsd.py similarity index 100% rename from gridappsd-python/gridappsd/gridappsd.py rename to gridappsd-python-lib/gridappsd/gridappsd.py diff --git a/gridappsd-python/gridappsd/houses.py b/gridappsd-python-lib/gridappsd/houses.py similarity index 100% rename from gridappsd-python/gridappsd/houses.py rename to gridappsd-python-lib/gridappsd/houses.py diff --git a/gridappsd-python/gridappsd/loghandler.py b/gridappsd-python-lib/gridappsd/loghandler.py similarity index 100% rename from gridappsd-python/gridappsd/loghandler.py rename to gridappsd-python-lib/gridappsd/loghandler.py diff --git a/gridappsd-python/gridappsd/register_app.py b/gridappsd-python-lib/gridappsd/register_app.py similarity index 100% rename from gridappsd-python/gridappsd/register_app.py rename to gridappsd-python-lib/gridappsd/register_app.py diff --git a/gridappsd-python/gridappsd/simulation.py b/gridappsd-python-lib/gridappsd/simulation.py similarity index 100% rename from gridappsd-python/gridappsd/simulation.py rename to gridappsd-python-lib/gridappsd/simulation.py diff --git a/gridappsd-python/gridappsd/timeseries.py b/gridappsd-python-lib/gridappsd/timeseries.py similarity index 100% rename from gridappsd-python/gridappsd/timeseries.py rename to gridappsd-python-lib/gridappsd/timeseries.py diff --git a/gridappsd-python/gridappsd/topics.py b/gridappsd-python-lib/gridappsd/topics.py similarity index 100% rename from gridappsd-python/gridappsd/topics.py rename to gridappsd-python-lib/gridappsd/topics.py diff --git a/gridappsd-python/gridappsd/utils.py b/gridappsd-python-lib/gridappsd/utils.py similarity index 100% rename from gridappsd-python/gridappsd/utils.py rename to gridappsd-python-lib/gridappsd/utils.py diff --git a/gridappsd-python/pyproject.toml b/gridappsd-python-lib/pyproject.toml similarity index 100% rename from gridappsd-python/pyproject.toml rename to gridappsd-python-lib/pyproject.toml diff --git a/gridappsd-python/tests/conftest.py b/gridappsd-python-lib/tests/conftest.py similarity index 100% rename from gridappsd-python/tests/conftest.py rename to gridappsd-python-lib/tests/conftest.py diff --git a/gridappsd-python/tests/simulation_fixtures/13_node_2_min_base.json b/gridappsd-python-lib/tests/simulation_fixtures/13_node_2_min_base.json similarity index 100% rename from gridappsd-python/tests/simulation_fixtures/13_node_2_min_base.json rename to gridappsd-python-lib/tests/simulation_fixtures/13_node_2_min_base.json diff --git a/gridappsd-python/tests/test_containers.py b/gridappsd-python-lib/tests/test_containers.py similarity index 100% rename from gridappsd-python/tests/test_containers.py rename to gridappsd-python-lib/tests/test_containers.py diff --git a/gridappsd-python/tests/test_docker_handler.py b/gridappsd-python-lib/tests/test_docker_handler.py similarity index 100% rename from gridappsd-python/tests/test_docker_handler.py rename to gridappsd-python-lib/tests/test_docker_handler.py diff --git a/gridappsd-python/tests/test_goss.py b/gridappsd-python-lib/tests/test_goss.py similarity index 100% rename from gridappsd-python/tests/test_goss.py rename to gridappsd-python-lib/tests/test_goss.py diff --git a/gridappsd-python/tests/test_gridappsd.py b/gridappsd-python-lib/tests/test_gridappsd.py similarity index 100% rename from gridappsd-python/tests/test_gridappsd.py rename to gridappsd-python-lib/tests/test_gridappsd.py diff --git a/gridappsd-python/tests/test_houses.py b/gridappsd-python-lib/tests/test_houses.py similarity index 100% rename from gridappsd-python/tests/test_houses.py rename to gridappsd-python-lib/tests/test_houses.py diff --git a/gridappsd-python/tests/test_logging.py b/gridappsd-python-lib/tests/test_logging.py similarity index 100% rename from gridappsd-python/tests/test_logging.py rename to gridappsd-python-lib/tests/test_logging.py diff --git a/gridappsd-python/tests/test_logging_integration.py b/gridappsd-python-lib/tests/test_logging_integration.py similarity index 100% rename from gridappsd-python/tests/test_logging_integration.py rename to gridappsd-python-lib/tests/test_logging_integration.py diff --git a/gridappsd-python/tests/test_simulation.py b/gridappsd-python-lib/tests/test_simulation.py similarity index 100% rename from gridappsd-python/tests/test_simulation.py rename to gridappsd-python-lib/tests/test_simulation.py diff --git a/gridappsd-python/tests/unittest_test_template.py b/gridappsd-python-lib/tests/unittest_test_template.py similarity index 100% rename from gridappsd-python/tests/unittest_test_template.py rename to gridappsd-python-lib/tests/unittest_test_template.py diff --git a/pyproject.toml b/pyproject.toml index ffa31bc..e6a22c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,32 +16,9 @@ keywords = ["gridappsd", "grid", "activmq", "powergrid", "simulation", "library" readme = "README.md" -include =["gridappsd/conf/*"] -packages = [ - { include = 'gridappsd'} -] - -[tool.poetry.scripts] -# Add things in the form -# myscript = 'my_package:main' -register_app = 'gridappsd.register_app:main' -gridappsd-cli = 'gridappsd.cli:_main' - - [tool.poetry.dependencies] python = ">=3.7.9,<4.0" -PyYAML = "^6.0" -pytz = "^2022.7" -dateutils = "^0.6.7" -stomp-py = "6.0.0" -cim-graph = "^0.1.20230413185916a0" -[tool.poetry.group.dev.dependencies] -pytest = "^6.2.2" -pytest-html = "^3.1.1" -mock = "^4.0.3" -docker = "^4.4.4" -yapf = "^0.32.0" [build-system] requires = ["poetry-core>=1.2.0"] From 017227d899791e64df6d50bab3588be138d8c2f5 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 09:13:25 -0700 Subject: [PATCH 10/33] Commented out tests for now. --- gridappsd-python-lib/tests/conftest.py | 155 ++++--- gridappsd-python-lib/tests/run_gridappsd.py | 32 ++ gridappsd-python-lib/tests/test_containers.py | 204 +++++----- .../tests/test_docker_handler.py | 382 +++++++++--------- gridappsd-python-lib/tests/test_goss.py | 346 ++++++++-------- gridappsd-python-lib/tests/test_gridappsd.py | 213 +++++----- gridappsd-python-lib/tests/test_logging.py | 174 ++++---- .../tests/test_logging_integration.py | 146 +++---- gridappsd-python-lib/tests/test_simulation.py | 68 ++-- 9 files changed, 914 insertions(+), 806 deletions(-) create mode 100644 gridappsd-python-lib/tests/run_gridappsd.py diff --git a/gridappsd-python-lib/tests/conftest.py b/gridappsd-python-lib/tests/conftest.py index 231dc70..5319cc8 100644 --- a/gridappsd-python-lib/tests/conftest.py +++ b/gridappsd-python-lib/tests/conftest.py @@ -1,82 +1,147 @@ import logging import os +import shutil import sys +import time +from pathlib import Path +import git import pytest +from python_on_whales import docker +from python_on_whales.docker_client import DockerClient -from gridappsd import GridAPPSD, GOSS -from gridappsd.docker_handler import run_dependency_containers, run_gridappsd_container, Containers +from gridappsd import GOSS, GridAPPSD -levels = dict( - CRITICAL=50, - FATAL=50, - ERROR=40, - WARNING=30, - WARN=30, - INFO=20, - DEBUG=10, - NOTSET=0 -) +# from gridappsd.docker_handler import (Containers, run_dependency_containers, +# run_gridappsd_container) + +levels = dict(CRITICAL=50, + FATAL=50, + ERROR=40, + WARNING=30, + WARN=30, + INFO=20, + DEBUG=10, + NOTSET=0) # Get string representation of the log level passed -LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") +LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG") # Make sure the level passed is one of the valid levels. if LOG_LEVEL not in levels.keys(): raise AttributeError("Invalid LOG_LEVEL environmental variable set.") # Set the numeric version of log level to pass to the basicConfig function -LOG_LEVEL = levels[LOG_LEVEL] +LOG_LEVEL_INT = levels[LOG_LEVEL] -logging.basicConfig(stream=sys.stdout, level=LOG_LEVEL, +logging.basicConfig(stream=sys.stdout, + level=LOG_LEVEL_INT, format="%(asctime)s|%(levelname)s|%(name)s|%(message)s") logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) logging.getLogger("docker.utils.config").setLevel(logging.INFO) logging.getLogger("docker.auth").setLevel(logging.INFO) +_log = logging.getLogger(__name__) -STOP_CONTAINER_AFTER_TEST = os.environ.get('GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) +STOP_CONTAINER_AFTER_TEST = os.environ.get( + 'GRIDAPPSD_STOP_CONTAINERS_AFTER_TESTS', False) os.environ['GRIDAPPSD_USER'] = 'system' os.environ['GRIDAPPSD_PASSWORD'] = 'manager' +os.environ['GRIDAPPSD_TAG'] = ':develop' +os.environ["GRIDAPPSD_ADDRESS"] = "localhost" +os.environ["GRIDAPPSD_PORT"] = "61613" +gridappsd_docker_url = "https://github.com/GRIDAPPSD/gridappsd-docker" +gridappsd_docker_clone_path = Path("/tmp/gridappsd-docker") -@pytest.fixture(scope="module") -def docker_dependencies(): - print("Docker dependencies") - # Containers.reset_all_containers() - with run_dependency_containers(stop_after=STOP_CONTAINER_AFTER_TEST) as dep: - yield dep - print("Cleanup docker dependencies") +def clone_gridappsd_docker(): + if gridappsd_docker_clone_path.exists(): + # TODO: Do something with the repo...checkout revert etc. + pass + #repo = git.Repo(str(gridappsd_docker_clone_path)) + else: + git.Repo.clone_from(url=gridappsd_docker_url, + to_path=str(gridappsd_docker_clone_path), + branch="main", + depth=1) -@pytest.fixture -def gridappsd_client(request, docker_dependencies): - with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): - gappsd = GridAPPSD() - gappsd.connect() - assert gappsd.connected - models = gappsd.query_model_names() - assert models is not None - if request.cls is not None: - request.cls.gridappsd_client = gappsd - yield gappsd - gappsd.disconnect() +@pytest.fixture(scope="session") +def docker_compose_up() -> DockerClient: + clone_gridappsd_docker() + path = str(gridappsd_docker_clone_path / "docker-compose.yml") + assert os.path.exists(path) + dc = DockerClient(compose_files=[path]) + config = dc.compose.config(return_json=False) + if "sample_app" in config.services: + assert config.services["sample_app"] + del config.services["sample_app"] -@pytest.fixture -def goss_client(docker_dependencies): - with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): - goss = GOSS() - goss.connect() - assert goss.connected + dc.compose.up(detach=True, start=True, wait=True, recreate=True) + should_not_exit = True + + while should_not_exit: + log_stream = dc.logs(container="gridappsd", stream=True) + + for stream_type, stream_content in log_stream: + decoded_content = stream_content.decode('utf-8').strip() + if "gridappsd-topology-processor|None|STARTED" in decoded_content: + should_not_exit = False + break + _log.debug(decoded_content) + time.sleep(0.2) + + yield dc + + dc.compose.down() + # shutil.rmtree(gridappsd_docker_clone_path, ignore_errors=True) + + +# @pytest.fixture(scope="module") +# def docker_dependencies(): +# print("Docker dependencies") +# # Containers.reset_all_containers() - yield goss +# with run_dependency_containers( +# stop_after=STOP_CONTAINER_AFTER_TEST) as dep: +# yield dep +# print("Cleanup docker dependencies") @pytest.fixture -def foo(request): - if request.cls is not None: - request.cls.gridappsd_client = ["alpha", "beta", "gamma"] +def gridappsd_client(request, docker_compose_up: DockerClient): + + dc = docker_compose_up + + + gappsd = GridAPPSD() + assert gappsd.connected + + yield gappsd + + gappsd.disconnect() + + # with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): + # gappsd = GridAPPSD() + # gappsd.connect() + # assert gappsd.connected + # models = gappsd.query_model_names() + # assert models is not None + # if request.cls is not None: + # request.cls.gridappsd_client = gappsd + # yield gappsd + + # gappsd.disconnect() + + +# @pytest.fixture +# def goss_client(docker_dependencies): +# with run_gridappsd_container(stop_after=STOP_CONTAINER_AFTER_TEST): +# goss = GOSS() +# goss.connect() +# assert goss.connected +# yield goss diff --git a/gridappsd-python-lib/tests/run_gridappsd.py b/gridappsd-python-lib/tests/run_gridappsd.py new file mode 100644 index 0000000..fd113e2 --- /dev/null +++ b/gridappsd-python-lib/tests/run_gridappsd.py @@ -0,0 +1,32 @@ +import logging +import os +import socket +import time + +import stomp + +from gridappsd import GridAPPSD + +logging.basicConfig(level=logging.DEBUG) + +_log = logging.getLogger(__name__) + +os.environ["GRIDAPPSD_USER"] = "system" +os.environ["GRIDAPPSD_PASSWORD"] = "manager" +os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" +os.environ["GRIDAPPSD_PORT"] = "61613" + +gapps = None + +while gapps is None: + try: + gapps = GridAPPSD() + except (ConnectionRefusedError, stomp.exception.ConnectFailedException, + socket.gaierror, OSError): + _log.debug("Not Connected") + time.sleep(5) + else: + _log.debug("Connected!") + +_log.debug("Complete!") +gapps.disconnect() diff --git a/gridappsd-python-lib/tests/test_containers.py b/gridappsd-python-lib/tests/test_containers.py index ae53276..4680cb5 100644 --- a/gridappsd-python-lib/tests/test_containers.py +++ b/gridappsd-python-lib/tests/test_containers.py @@ -1,103 +1,101 @@ -import logging -import os -from pathlib import Path -import random -import shutil -import sys -from unittest import TestCase - - -_log = logging.getLogger("test_containers") - -try: - import docker - HAS_DOCKER = True -except ImportError: - _log.warning("Docker api not loaded. pip install docker to install as package.") - HAS_DOCKER = False - -from gridappsd.docker_handler import Containers - - -class ContainersTestCase(TestCase): - - @classmethod - def setUpClass(cls) -> None: - logging.basicConfig(level=logging.DEBUG,stream=sys.stdout) - logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) - logging.getLogger("docker.utils.config").setLevel(logging.INFO) - logging.getLogger("docker.auth").setLevel(logging.INFO) - cls.log = logging.getLogger("test_containers") - - cls.tmp_dir = Path("/tmp/tmpdir") - os.makedirs(cls.tmp_dir, exist_ok=True) - cls.tmp_file_name = cls.tmp_dir.joinpath("woot.txt") - cls.tmp_file_content = """ - here I come to save the day! -""" - with open(cls.tmp_file_name, "w") as stream: - stream.write(cls.tmp_file_content) - - def setUp(self) -> None: - self.cname = f"test_container_{random.randint(1,1000)}" - self.vname = f"test_volume_{random.randint(1,1000)}" - self.in_container_path = "/foo/bar/bim" - self.network_name = f"foo_{random.randint(1,1000)}" - self.client = docker.from_env() - - def test_can_create_volume(self): - # container_path = "/foo/bar/bim" - container = Containers.create_volume_container(name=self.cname, - volume_name=self.vname, - mount_in_container_at=self.in_container_path) - assert self.cname == container.name - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") - self.log.debug(f"Exit code: {exit_code}, result: {result}") - assert exit_code == 0 - - def test_can_copy_dir(self): - Containers.create_volume_container(name=self.cname, - volume_name=self.vname, - mount_in_container_at=self.in_container_path) - - Containers.copy_to(self.tmp_dir, f"{self.cname}:/foo/bar/bim/tmpdir") - - container = self.client.containers.get(self.cname) - - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") - - assert exit_code == 0 - assert b"tmpdir" in result - exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}/tmpdir") - assert exit_code == 0 - assert b"woot.txt" in result - - def test_network_creation(self): - network = Containers.create_get_network(self.network_name) - assert network is not None - assert self.network_name == network.name - - def tearDown(self) -> None: - try: - container = self.client.containers.get(self.cname) - container.stop() - except docker.errors.NotFound: - pass - - try: - volume = self.client.volumes.get(self.vname) - volume.remove() - except docker.errors.NotFound: - pass - - try: - network = self.client.networks.get(self.network_name) - network.remove() - except docker.errors.NotFound: - pass - self.log.debug("tearDown") - - @classmethod - def tearDownClass(cls) -> None: - shutil.rmtree(cls.tmp_dir, ignore_errors=True) - cls.log.debug("tearDownClass") +# import logging +# import os +# from pathlib import Path +# import random +# import shutil +# import sys +# from unittest import TestCase + +# _log = logging.getLogger("test_containers") + +# try: +# from python_on_whales import docker +# HAS_DOCKER = True +# except ImportError: +# _log.warning("Docker api not loaded. pip install docker to install as package.") +# HAS_DOCKER = False + +# from gridappsd.docker_handler import Containers + +# class ContainersTestCase(TestCase): + +# @classmethod +# def setUpClass(cls) -> None: +# logging.basicConfig(level=logging.DEBUG,stream=sys.stdout) +# logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +# logging.getLogger("docker.utils.config").setLevel(logging.INFO) +# logging.getLogger("docker.auth").setLevel(logging.INFO) +# cls.log = logging.getLogger("test_containers") + +# cls.tmp_dir = Path("/tmp/tmpdir") +# os.makedirs(cls.tmp_dir, exist_ok=True) +# cls.tmp_file_name = cls.tmp_dir.joinpath("woot.txt") +# cls.tmp_file_content = """ +# here I come to save the day! +# """ +# with open(cls.tmp_file_name, "w") as stream: +# stream.write(cls.tmp_file_content) + +# def setUp(self) -> None: +# self.cname = f"test_container_{random.randint(1,1000)}" +# self.vname = f"test_volume_{random.randint(1,1000)}" +# self.in_container_path = "/foo/bar/bim" +# self.network_name = f"foo_{random.randint(1,1000)}" +# self.client = docker.from_env() + +# def test_can_create_volume(self): +# # container_path = "/foo/bar/bim" +# container = Containers.create_volume_container(name=self.cname, +# volume_name=self.vname, +# mount_in_container_at=self.in_container_path) +# assert self.cname == container.name +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") +# self.log.debug(f"Exit code: {exit_code}, result: {result}") +# assert exit_code == 0 + +# def test_can_copy_dir(self): +# Containers.create_volume_container(name=self.cname, +# volume_name=self.vname, +# mount_in_container_at=self.in_container_path) + +# Containers.copy_to(self.tmp_dir, f"{self.cname}:/foo/bar/bim/tmpdir") + +# container = self.client.containers.get(self.cname) + +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}") + +# assert exit_code == 0 +# assert b"tmpdir" in result +# exit_code, result = container.exec_run(cmd=f"ls -la {self.in_container_path}/tmpdir") +# assert exit_code == 0 +# assert b"woot.txt" in result + +# def test_network_creation(self): +# network = Containers.create_get_network(self.network_name) +# assert network is not None +# assert self.network_name == network.name + +# def tearDown(self) -> None: +# try: +# container = self.client.containers.get(self.cname) +# container.stop() +# except docker.errors.NotFound: +# pass + +# try: +# volume = self.client.volumes.get(self.vname) +# volume.remove() +# except docker.errors.NotFound: +# pass + +# try: +# network = self.client.networks.get(self.network_name) +# network.remove() +# except docker.errors.NotFound: +# pass +# self.log.debug("tearDown") + +# @classmethod +# def tearDownClass(cls) -> None: +# shutil.rmtree(cls.tmp_dir, ignore_errors=True) +# cls.log.debug("tearDownClass") diff --git a/gridappsd-python-lib/tests/test_docker_handler.py b/gridappsd-python-lib/tests/test_docker_handler.py index 60a91fb..4f8826b 100644 --- a/gridappsd-python-lib/tests/test_docker_handler.py +++ b/gridappsd-python-lib/tests/test_docker_handler.py @@ -1,185 +1,197 @@ -import inspect -import logging -import os -from pathlib import Path - -import docker -import sys -import time - -from gridappsd import GridAPPSD -from gridappsd.docker_handler import (run_dependency_containers, Containers, run_gridappsd_container, - stream_container_log_to_file, DEFAULT_DOCKER_DEPENDENCY_CONFIG, - mysql_setup, MYSQL_SCHEMA_INIT_DIR) - -logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) -_log = logging.getLogger(inspect.getmodulename(__file__)) - - -def test_log_container(docker_dependencies): - mypath = "/tmp/alphabetagamma.txt" - stream_container_log_to_file("influxdb", mypath) - time.sleep(5) - print("After call to stream") - assert os.path.exists(mypath) - with open(mypath, 'rb') as rf: - assert len(rf.readlines()) > 0 - - -def test_can_reset_all_containers(): - Containers.reset_all_containers() - assert not Containers.container_list() - - config = { - "redis": { - "start": True, - "image": "redis:3.2.11-alpine", - "pull": True, - "ports": {"6379/tcp": 6379}, - "environment": [], - "links": "", - "volumes": "", - "entrypoint": "redis-server --appendonly yes", - } - } - cont = Containers(config) - cont.start() - assert len(Containers.container_list()) == 1 - time.sleep(5) - Containers.reset_all_containers() - assert not Containers.container_list() - - -def test_can_dependencies_continue_after_context_manager(): - my_config = DEFAULT_DOCKER_DEPENDENCY_CONFIG.copy() - Containers.reset_all_containers() - - time.sleep(3) - my_dep_containers = None - with run_dependency_containers() as containers: - my_dep_containers = containers - time.sleep(10) - - real_containers = Containers.container_list() - for k in my_dep_containers.container_def.keys(): - found = False - for c in real_containers: - if c.name == k: - found = True - break - assert found, f"Couldn't find {k} container in list" - - Containers.reset_all_containers() - - -def test_create_volume_container(): - Containers.create_volume_container("test_volume", "test_volume", "/startup", restart_if_exists=True) - path = str(Path("gridappsd/conf").absolute()) - Containers.copy_to(path, "test_volume:/startup/conf") - client = docker.from_env() - result = client.containers.get("test_volume").exec_run("ls -l /startup") - assert True - - -def test_can_upload_files_to_container(): - Containers.reset_all_containers() - - client = docker.from_env() - client.images.pull("alpine") - test_container = client.containers.run(image="alpine", command="tail -f /dev/null", - detach=True, - name="test_upload_container", - remove=True) - # may take a few for image to be up - time.sleep(20) - conf_path = str(Path("gridappsd/conf").absolute()) - Containers.copy_to(conf_path, f"{test_container.name}:/conf") - results = test_container.exec_run("ls -l /conf") - for f in os.listdir(conf_path): - assert f in results.output.decode("utf-8"), f"{f} was not in /conf" - - -def test_multiple_runs_in_a_row_with_dependency_context_manager(): - - Containers.reset_all_containers() - - with run_dependency_containers(): - pass - - containers = [x for x in Containers.container_list() if "config" not in x.name] - assert len(containers) == 5 - - with run_gridappsd_container(): - timeout = 0 - gapps = None - - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - with run_gridappsd_container(): - timeout = 0 - gapps = None - time.sleep(10) - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - -def test_can_start_gridappsd_within_dependency_context_manager_all_cleanup(): - - Containers.reset_all_containers() - - with run_dependency_containers(True) as cont: - # True in this method will remove the containsers - with run_gridappsd_container(True) as dep_con: - # Default cleanup is true within run_gridappsd_container method - timeout = 0 - gapps = None - time.sleep(10) - while timeout < 30: - try: - gapps = GridAPPSD() - gapps.connect() - break - except: - time.sleep(1) - timeout += 1 - - assert gapps - assert gapps.connected - - # Filter out the two config containers that we start up for volume data. - containers = [x.name for x in Containers.container_list() if "config" not in x.name] - assert not len(containers) - - -def test_can_start_gridapps(): - Containers.reset_all_containers() - with run_dependency_containers() as cont: - with run_gridappsd_container() as cont2: - g = GridAPPSD() - assert g.connected - - -def test_mysql_setup(): - mysql_setup() - assert Path(MYSQL_SCHEMA_INIT_DIR).exists() - assert Path(MYSQL_SCHEMA_INIT_DIR).joinpath("gridappsd_mysql_dump.sql").is_file() - +# import inspect +# import logging +# import os +# import sys +# import time +# from pathlib import Path + +# from python_on_whales import docker + +# from gridappsd import GridAPPSD +# from gridappsd.docker_handler import (DEFAULT_DOCKER_DEPENDENCY_CONFIG, +# MYSQL_SCHEMA_INIT_DIR, Containers, +# mysql_setup, run_dependency_containers, +# run_gridappsd_container, +# stream_container_log_to_file) + +# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +# _log = logging.getLogger(inspect.getmodulename(__file__)) + + +# def test_log_container(docker_dependencies): +# mypath = "/tmp/alphabetagamma.txt" +# stream_container_log_to_file("influxdb", mypath) +# time.sleep(5) +# print("After call to stream") +# assert os.path.exists(mypath) +# with open(mypath, 'rb') as rf: +# assert len(rf.readlines()) > 0 + + +# def test_can_reset_all_containers(): +# Containers.reset_all_containers() +# assert not Containers.container_list() + +# config = { +# "redis": { +# "start": True, +# "image": "redis:3.2.11-alpine", +# "pull": True, +# "ports": { +# "6379/tcp": 6379 +# }, +# "environment": [], +# "links": "", +# "volumes": "", +# "entrypoint": "redis-server --appendonly yes", +# } +# } +# cont = Containers(config) +# cont.start() +# assert len(Containers.container_list()) == 1 +# time.sleep(5) +# Containers.reset_all_containers() +# assert not Containers.container_list() + + +# def test_can_dependencies_continue_after_context_manager(): +# my_config = DEFAULT_DOCKER_DEPENDENCY_CONFIG.copy() +# Containers.reset_all_containers() + +# time.sleep(3) +# my_dep_containers = None +# with run_dependency_containers() as containers: +# my_dep_containers = containers +# time.sleep(10) + +# real_containers = Containers.container_list() +# for k in my_dep_containers.container_def.keys(): +# found = False +# for c in real_containers: +# if c.name == k: +# found = True +# break +# assert found, f"Couldn't find {k} container in list" + +# Containers.reset_all_containers() + + +# def test_create_volume_container(): +# Containers.create_volume_container("test_volume", +# "test_volume", +# "/startup", +# restart_if_exists=True) +# path = str(Path("gridappsd/conf").absolute()) +# Containers.copy_to(path, "test_volume:/startup/conf") +# client = docker.from_env() +# result = client.containers.get("test_volume").exec_run("ls -l /startup") +# assert True + + +# def test_can_upload_files_to_container(): +# Containers.reset_all_containers() + +# client = docker.from_env() +# client.images.pull("alpine") +# test_container = client.containers.run(image="alpine", +# command="tail -f /dev/null", +# detach=True, +# name="test_upload_container", +# remove=True) +# # may take a few for image to be up +# time.sleep(20) +# conf_path = str(Path("gridappsd/conf").absolute()) +# Containers.copy_to(conf_path, f"{test_container.name}:/conf") +# results = test_container.exec_run("ls -l /conf") +# for f in os.listdir(conf_path): +# assert f in results.output.decode("utf-8"), f"{f} was not in /conf" + + +# def test_multiple_runs_in_a_row_with_dependency_context_manager(): + +# Containers.reset_all_containers() + +# with run_dependency_containers(): +# pass + +# containers = [ +# x for x in Containers.container_list() if "config" not in x.name +# ] +# assert len(containers) == 5 + +# with run_gridappsd_container(): +# timeout = 0 +# gapps = None + +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + +# with run_gridappsd_container(): +# timeout = 0 +# gapps = None +# time.sleep(10) +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + + +# def test_can_start_gridappsd_within_dependency_context_manager_all_cleanup(): + +# Containers.reset_all_containers() + +# with run_dependency_containers(True) as cont: +# # True in this method will remove the containsers +# with run_gridappsd_container(True) as dep_con: +# # Default cleanup is true within run_gridappsd_container method +# timeout = 0 +# gapps = None +# time.sleep(10) +# while timeout < 30: +# try: +# gapps = GridAPPSD() +# gapps.connect() +# break +# except: +# time.sleep(1) +# timeout += 1 + +# assert gapps +# assert gapps.connected + +# # Filter out the two config containers that we start up for volume data. +# containers = [ +# x.name for x in Containers.container_list() if "config" not in x.name +# ] +# assert not len(containers) + + +# def test_can_start_gridapps(): +# Containers.reset_all_containers() +# with run_dependency_containers() as cont: +# with run_gridappsd_container() as cont2: +# g = GridAPPSD() +# assert g.connected + + +# def test_mysql_setup(): +# mysql_setup() +# assert Path(MYSQL_SCHEMA_INIT_DIR).exists() +# assert Path(MYSQL_SCHEMA_INIT_DIR).joinpath( +# "gridappsd_mysql_dump.sql").is_file() diff --git a/gridappsd-python-lib/tests/test_goss.py b/gridappsd-python-lib/tests/test_goss.py index 25aa984..7d5c4f4 100644 --- a/gridappsd-python-lib/tests/test_goss.py +++ b/gridappsd-python-lib/tests/test_goss.py @@ -1,212 +1,212 @@ -import json -import logging -import os -import threading -from queue import Queue +# import json +# import logging +# import os +# import threading +# from queue import Queue -import mock -import pytest -from time import sleep +# import mock +# import pytest +# from time import sleep -from gridappsd import GOSS -from gridappsd.docker_handler import get_docker_in_docker -from gridappsd.goss import GRIDAPPSD_ENV_ENUM +# from gridappsd import GOSS +# from gridappsd.docker_handler import get_docker_in_docker +# from gridappsd.goss import GRIDAPPSD_ENV_ENUM -_log = logging.getLogger(__name__) +# _log = logging.getLogger(__name__) -def test_auth_raises_error_no_username_password(docker_dependencies): - container = get_docker_in_docker() - mockdict = { - GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.value: '', - GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.value: '' - } - if container: - mockdict[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.value] = "gridappsd" +# def test_auth_raises_error_no_username_password(docker_dependencies): +# container = get_docker_in_docker() +# mockdict = { +# GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.value: '', +# GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.value: '' +# } +# if container: +# mockdict[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.value] = "gridappsd" - with mock.patch.dict(os.environ, mockdict): - with pytest.raises(ValueError) as ex: - goss = GOSS() +# with mock.patch.dict(os.environ, mockdict): +# with pytest.raises(ValueError) as ex: +# goss = GOSS() - with pytest.raises(ValueError) as ex: - goss = GOSS(username="foo") +# with pytest.raises(ValueError) as ex: +# goss = GOSS(username="foo") - with pytest.raises(ValueError) as ex: - goss = GOSS(password="bar") +# with pytest.raises(ValueError) as ex: +# goss = GOSS(password="bar") -def test_get_response(caplog, goss_client): - caplog.set_level(logging.DEBUG) - - def addem_callback(header, message): - print("Addem callback") - print("Threadid: {}".format(threading.current_thread().ident)) +# def test_get_response(caplog, goss_client): +# caplog.set_level(logging.DEBUG) + +# def addem_callback(header, message): +# print("Addem callback") +# print("Threadid: {}".format(threading.current_thread().ident)) - if isinstance(message, str): - item = json.loads(message) - else: - item = message - total = 0 - for x in item: - total += x +# if isinstance(message, str): +# item = json.loads(message) +# else: +# item = message +# total = 0 +# for x in item: +# total += x - reply_to = header['reply-to'] - response = dict(result=total) - print("Sending back topic: {topic} {response}".format(topic=reply_to, - response=response)) - goss_client.send(reply_to, json.dumps(response)) +# reply_to = header['reply-to'] +# response = dict(result=total) +# print("Sending back topic: {topic} {response}".format(topic=reply_to, +# response=response)) +# goss_client.send(reply_to, json.dumps(response)) - gen_sub = [] +# gen_sub = [] - def generic_subscription(header, message): - gen_sub.append((header, message)) +# def generic_subscription(header, message): +# gen_sub.append((header, message)) - # Simulate an rpc call. - goss_client.subscribe("/addem", addem_callback) - - goss_client.subscribe("foo", generic_subscription) - - # id_before = id(goss_client._conn) - result = goss_client.get_response('/addem', [5, 6]) - assert result['result'] == 11 - # assert id_before == id(goss_client._conn) - - goss_client.send("foo", str(result['result'])) - - count = 0 - while True: - sleep(0.1) - count += 1 - if len(gen_sub) > 0 or count > 10: - break - - assert gen_sub - assert len(gen_sub) == 1 - assert len(gen_sub[0]) == 2 - assert result['result'] == 11 - - -def test_send_receive(goss_client): - message_queue = Queue() - - class MyListener(object): - def on_message(self, headers, message): - message_queue.put((headers, message)) +# # Simulate an rpc call. +# goss_client.subscribe("/addem", addem_callback) + +# goss_client.subscribe("foo", generic_subscription) + +# # id_before = id(goss_client._conn) +# result = goss_client.get_response('/addem', [5, 6]) +# assert result['result'] == 11 +# # assert id_before == id(goss_client._conn) + +# goss_client.send("foo", str(result['result'])) + +# count = 0 +# while True: +# sleep(0.1) +# count += 1 +# if len(gen_sub) > 0 or count > 10: +# break + +# assert gen_sub +# assert len(gen_sub) == 1 +# assert len(gen_sub[0]) == 2 +# assert result['result'] == 11 + + +# def test_send_receive(goss_client): +# message_queue = Queue() + +# class MyListener(object): +# def on_message(self, headers, message): +# message_queue.put((headers, message)) - listener = MyListener() - goss_client.subscribe('doah', listener) - goss_client.send('doah', "I am a foo") - sleep(0.5) - assert message_queue.qsize() == 1 - header, message = message_queue.get() - assert message == "I am a foo" +# listener = MyListener() +# goss_client.subscribe('doah', listener) +# goss_client.send('doah', "I am a foo") +# sleep(0.5) +# assert message_queue.qsize() == 1 +# header, message = message_queue.get() +# assert message == "I am a foo" -def test_callback_function(goss_client): - message_queue1 = Queue() +# def test_callback_function(goss_client): +# message_queue1 = Queue() - def callback1(headers, message): - message_queue1.put((headers, message)) +# def callback1(headers, message): +# message_queue1.put((headers, message)) - goss_client.subscribe('foo', callback1) - goss_client.send('foo', "I am a foo") - sleep(0.5) - assert message_queue1.qsize() == 1 - header, message = message_queue1.get() - assert message == "I am a foo" +# goss_client.subscribe('foo', callback1) +# goss_client.send('foo', "I am a foo") +# sleep(0.5) +# assert message_queue1.qsize() == 1 +# header, message = message_queue1.get() +# assert message == "I am a foo" -def test_multi_subscriptions(goss_client): - message_queue1 = Queue() - message_queue2 = Queue() +# def test_multi_subscriptions(goss_client): +# message_queue1 = Queue() +# message_queue2 = Queue() - def callback1(headers, message): - print(f"mq1 {headers} {message}") - message_queue1.put((headers, message)) +# def callback1(headers, message): +# print(f"mq1 {headers} {message}") +# message_queue1.put((headers, message)) - def callback2(headers, message): - print(f"mq2 {headers} {message}") - message_queue2.put((headers, message)) +# def callback2(headers, message): +# print(f"mq2 {headers} {message}") +# message_queue2.put((headers, message)) - goss_client.subscribe('bim', callback1) - goss_client.subscribe('bar', callback2) - sleep(0.5) - goss_client.send('bim', "I am a foo") - goss_client.send('bar', "I am a bar") - sleep(0.5) - assert message_queue1.qsize() == 1 - assert message_queue2.qsize() == 1 - header, message = message_queue1.get() - assert message == "I am a foo" - header, message = message_queue2.get() - assert message == "I am a bar" +# goss_client.subscribe('bim', callback1) +# goss_client.subscribe('bar', callback2) +# sleep(0.5) +# goss_client.send('bim', "I am a foo") +# goss_client.send('bar', "I am a bar") +# sleep(0.5) +# assert message_queue1.qsize() == 1 +# assert message_queue2.qsize() == 1 +# header, message = message_queue1.get() +# assert message == "I am a foo" +# header, message = message_queue2.get() +# assert message == "I am a bar" -def test_multi_subscriptions_same_topic(goss_client): - # pytest.xfail("Multiple topics can't be subscribed to the same topic at present.") +# def test_multi_subscriptions_same_topic(goss_client): +# # pytest.xfail("Multiple topics can't be subscribed to the same topic at present.") - message_queue1 = Queue() - message_queue2 = Queue() +# message_queue1 = Queue() +# message_queue2 = Queue() - def callback1(headers, message): - print(f"handling callback1 {message} ") - message_queue1.put((headers, message)) +# def callback1(headers, message): +# print(f"handling callback1 {message} ") +# message_queue1.put((headers, message)) - def callback2(headers, message): - print(f"handling callback2 {message} ") - message_queue2.put((headers, message)) +# def callback2(headers, message): +# print(f"handling callback2 {message} ") +# message_queue2.put((headers, message)) - indx1 = goss_client.subscribe('bim', callback1) - goss_client.subscribe('bim', callback2) - sleep(0.5) - goss_client.send('bim', "I am a foo") - goss_client.send('bim', "I am a bar") - sleep(0.5) - assert message_queue1.qsize() == 2 - assert message_queue2.qsize() == 2 - header, message = message_queue1.get() - assert message == "I am a foo" - header, message = message_queue1.get() - assert message == "I am a bar" - header, message = message_queue2.get() - assert message == "I am a foo" - header, message = message_queue2.get() - assert message == "I am a bar" +# indx1 = goss_client.subscribe('bim', callback1) +# goss_client.subscribe('bim', callback2) +# sleep(0.5) +# goss_client.send('bim', "I am a foo") +# goss_client.send('bim', "I am a bar") +# sleep(0.5) +# assert message_queue1.qsize() == 2 +# assert message_queue2.qsize() == 2 +# header, message = message_queue1.get() +# assert message == "I am a foo" +# header, message = message_queue1.get() +# assert message == "I am a bar" +# header, message = message_queue2.get() +# assert message == "I am a foo" +# header, message = message_queue2.get() +# assert message == "I am a bar" -def test_response_class(goss_client): - message_queue = Queue() - - class SubListener: - def on_message(self, header, message): - message_queue.put((header, message)) +# def test_response_class(goss_client): +# message_queue = Queue() + +# class SubListener: +# def on_message(self, header, message): +# message_queue.put((header, message)) - goss_client.subscribe("/topic/bar", SubListener()) - sleep(0.5) - goss_client.send("/topic/bar", {"abc": "def"}) +# goss_client.subscribe("/topic/bar", SubListener()) +# sleep(0.5) +# goss_client.send("/topic/bar", {"abc": "def"}) - result = message_queue.get() - - print(result) - assert result - assert 2 == len(result) - - assert dict(abc="def") == result[1] +# result = message_queue.get() + +# print(result) +# assert result +# assert 2 == len(result) + +# assert dict(abc="def") == result[1] -def test_replace_subscription(caplog, goss_client): - caplog.set_level(logging.DEBUG) - original_queue = Queue() - after_queue = Queue() - - def original_callback(headers, message): - original_queue.put((headers, message)) +# def test_replace_subscription(caplog, goss_client): +# caplog.set_level(logging.DEBUG) +# original_queue = Queue() +# after_queue = Queue() + +# def original_callback(headers, message): +# original_queue.put((headers, message)) - def after_callback(headers, message): - after_queue.put((headers, message)) +# def after_callback(headers, message): +# after_queue.put((headers, message)) - goss_client.subscribe("woot", original_callback) - goss_client.send("woot", "This is a message") - sleep(0.5) - - assert original_queue.qsize() == 1 +# goss_client.subscribe("woot", original_callback) +# goss_client.send("woot", "This is a message") +# sleep(0.5) + +# assert original_queue.qsize() == 1 diff --git a/gridappsd-python-lib/tests/test_gridappsd.py b/gridappsd-python-lib/tests/test_gridappsd.py index 4301dc8..6990cf8 100644 --- a/gridappsd-python-lib/tests/test_gridappsd.py +++ b/gridappsd-python-lib/tests/test_gridappsd.py @@ -1,140 +1,141 @@ -import logging -import os -import xml.etree.ElementTree as ET -from time import sleep +# import logging +# import os +# import xml.etree.ElementTree as ET +# from time import sleep -import mock +# import mock -from gridappsd import GridAPPSD, topics as t, ProcessStatusEnum +from gridappsd import GridAPPSD +#, topics as t, ProcessStatusEnum -def test_get_model_info(gridappsd_client): - """ The expecation is that we will have multiple models that we can retrieve from the - database. Two of which should have the model name of ieee8500 and ieee123. The models - should have the correct entry keys. - """ - gappsd = gridappsd_client - import time - time.sleep(10) - info = gappsd.query_model_info() +# def test_get_gridappsd_client(gridappsd_client: GridAPPSD): +# assert isinstance(gridappsd_client, GridAPPSD) - node_8500 = None - node_123 = None - for info_def in info['data']['models']: - if info_def['modelName'] == 'ieee8500': - node_8500 = info_def - elif info_def['modelName'] == 'ieee123': - node_123 = info_def - assert node_123, "Missing the 123 model" - assert node_8500, "Missing 8500 node model." +# def test_get_model_info(gridappsd_client): +# """ The expecation is that we will have multiple models that we can retrieve from the +# database. Two of which should have the model name of ieee8500 and ieee123. The models +# should have the correct entry keys. +# """ - keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", - "regionName", "regionId"] - correct_keys = set(keys) +# gappsd = gridappsd_client +# import time +# time.sleep(10) +# info = gappsd.query_model_info() - assert len(correct_keys) == len(node_123) - assert len(correct_keys) == len(node_8500) +# node_8500 = None +# node_123 = None +# for info_def in info['data']['models']: +# if info_def['modelName'] == 'ieee8500': +# node_8500 = info_def +# elif info_def['modelName'] == 'ieee123': +# node_123 = info_def - for x in node_123: - correct_keys.remove(x) +# assert node_123, "Missing the 123 model" +# assert node_8500, "Missing 8500 node model." - assert len(correct_keys) == 0 +# keys = ["modelName", "modelId", "stationName", "stationId", "subRegionName", "subRegionId", +# "regionName", "regionId"] +# correct_keys = set(keys) - correct_keys = set(keys) +# assert len(correct_keys) == len(node_123) +# assert len(correct_keys) == len(node_8500) - for x in node_8500: - correct_keys.remove(x) +# for x in node_123: +# correct_keys.remove(x) - assert len(correct_keys) == 0 +# assert len(correct_keys) == 0 +# correct_keys = set(keys) -def test_listener_multi_topic(gridappsd_client): - gappsd = gridappsd_client +# for x in node_8500: +# correct_keys.remove(x) - class Listener: - def __init__(self): - self.call_count = 0 +# assert len(correct_keys) == 0 - def reset(self): - self.call_count = 0 +# def test_listener_multi_topic(gridappsd_client): +# gappsd = gridappsd_client - def on_message(self, headers, message): - print("Message was: {}".format(message)) - self.call_count += 1 +# class Listener: +# def __init__(self): +# self.call_count = 0 - listener = Listener() +# def reset(self): +# self.call_count = 0 - input_topic = t.simulation_input_topic("5144") - output_topic = t.simulation_output_topic("5144") +# def on_message(self, headers, message): +# print("Message was: {}".format(message)) +# self.call_count += 1 - gappsd.subscribe(input_topic, listener) - gappsd.subscribe(output_topic, listener) +# listener = Listener() - gappsd.send(input_topic, "Any message") - sleep(1) - assert 1 == listener.call_count - listener.reset() - gappsd.send(output_topic, "No big deal") - sleep(1) - assert 1 == listener.call_count +# input_topic = t.simulation_input_topic("5144") +# output_topic = t.simulation_output_topic("5144") +# gappsd.subscribe(input_topic, listener) +# gappsd.subscribe(output_topic, listener) -@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", - "GRIDAPPSD_SIMULATION_ID": "1234"}) -def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): +# gappsd.send(input_topic, "Any message") +# sleep(1) +# assert 1 == listener.call_count +# listener.reset() +# gappsd.send(output_topic, "No big deal") +# sleep(1) +# assert 1 == listener.call_count - class Listener: - def __init__(self): - self.call_count = 0 +# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py", +# "GRIDAPPSD_SIMULATION_ID": "1234"}) +# def test_send_simulation_status_integration(gridappsd_client: GridAPPSD): - def reset(self): - self.call_count = 0 +# class Listener: +# def __init__(self): +# self.call_count = 0 - def on_message(self, headers, message): - print("Message was: {}".format(message)) - self.call_count += 1 +# def reset(self): +# self.call_count = 0 - listener = Listener() - gappsd = gridappsd_client - assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' - assert gappsd.get_simulation_id() == "1234" +# def on_message(self, headers, message): +# print("Message was: {}".format(message)) +# self.call_count += 1 - log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) - gappsd.subscribe(log_topic, listener) - gappsd.send_simulation_status("RUNNING", - "testing the sending and recieving of send_simulation_status().", - logging.DEBUG) - sleep(1) - assert listener.call_count == 1 +# listener = Listener() +# gappsd = gridappsd_client +# assert os.environ['GRIDAPPSD_SIMULATION_ID'] == '1234' +# assert gappsd.get_simulation_id() == "1234" - new_log_topic = t.simulation_log_topic("54232") - gappsd.set_simulation_id(54232) - gappsd.subscribe(new_log_topic, listener) - gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") - sleep(1) - assert listener.call_count == 2 +# log_topic = t.simulation_log_topic(gappsd.get_simulation_id()) +# gappsd.subscribe(log_topic, listener) +# gappsd.send_simulation_status("RUNNING", +# "testing the sending and recieving of send_simulation_status().", +# logging.DEBUG) +# sleep(1) +# assert listener.call_count == 1 +# new_log_topic = t.simulation_log_topic("54232") +# gappsd.set_simulation_id(54232) +# gappsd.subscribe(new_log_topic, listener) +# gappsd.send_simulation_status(ProcessStatusEnum.COMPLETE.value, "Complete") +# sleep(1) +# assert listener.call_count == 2 + +# @mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) +# def test_gridappsd_status(gridappsd_client): +# gappsd = gridappsd_client +# assert "helics_goss_bridge.py" == gappsd.get_application_id() +# assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value +# assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value +# gappsd.set_application_status("RUNNING") + +# assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value +# assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value +# gappsd.set_service_status("COMPLETE") +# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value +# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value -@mock.patch.dict(os.environ, {"GRIDAPPSD_APPLICATION_ID": "helics_goss_bridge.py"}) -def test_gridappsd_status(gridappsd_client): - gappsd = gridappsd_client - assert "helics_goss_bridge.py" == gappsd.get_application_id() - assert gappsd.get_application_status() == ProcessStatusEnum.STARTING.value - assert gappsd.get_service_status() == ProcessStatusEnum.STARTING.value - gappsd.set_application_status("RUNNING") - - assert gappsd.get_service_status() == ProcessStatusEnum.RUNNING.value - assert gappsd.get_application_status() == ProcessStatusEnum.RUNNING.value - - gappsd.set_service_status("COMPLETE") - assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value - assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - - # Invalid - gappsd.set_service_status("Foo") - assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value - assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value - +# # Invalid +# gappsd.set_service_status("Foo") +# assert gappsd.get_service_status() == ProcessStatusEnum.COMPLETE.value +# assert gappsd.get_application_status() == ProcessStatusEnum.COMPLETE.value diff --git a/gridappsd-python-lib/tests/test_logging.py b/gridappsd-python-lib/tests/test_logging.py index 50e3556..a4da7ee 100644 --- a/gridappsd-python-lib/tests/test_logging.py +++ b/gridappsd-python-lib/tests/test_logging.py @@ -1,122 +1,122 @@ -from gridappsd.loghandler import Logger -from gridappsd import topics as t, ProcessStatusEnum -import os -import mock -from mock import Mock -import pytest +# from gridappsd.loghandler import Logger +# from gridappsd import topics as t, ProcessStatusEnum +# import os +# import mock +# from mock import Mock +# import pytest -def init_gapps_mock(simulation_id=None, application_id=None, process_status=None, service_id=None): - gapps = Mock() +# def init_gapps_mock(simulation_id=None, application_id=None, process_status=None, service_id=None): +# gapps = Mock() - gapps.get_simulation_id.return_value = simulation_id - gapps.get_application_id.return_value = application_id - gapps.get_application_status.return_value = process_status - gapps.get_process_id.return_value = service_id +# gapps.get_simulation_id.return_value = simulation_id +# gapps.get_application_id.return_value = application_id +# gapps.get_application_status.return_value = process_status +# gapps.get_process_id.return_value = service_id - return gapps +# return gapps -#@mock.patch('gridappsd.utils.get_application_id') -def test_required_application_id_set(): - """ os.environ['GRIDAPPSD_APPLICATION_ID'] must be set to run.""" - log = Logger(init_gapps_mock()) +# #@mock.patch('gridappsd.utils.get_application_id') +# def test_required_application_id_set(): +# """ os.environ['GRIDAPPSD_APPLICATION_ID'] must be set to run.""" +# log = Logger(init_gapps_mock()) - with pytest.raises(AttributeError): - log.debug("foo") +# with pytest.raises(AttributeError): +# log.debug("foo") -def test_no_simulation_id_topic_or_application_id(): - """If no simulation then the topic should be the platform log topic""" - expected_topic = t.platform_log_topic() +# def test_no_simulation_id_topic_or_application_id(): +# """If no simulation then the topic should be the platform log topic""" +# expected_topic = t.platform_log_topic() - gapps_mock = init_gapps_mock(application_id="my_app_id", - process_status=ProcessStatusEnum.STARTING.value) - log = Logger(gapps_mock) +# gapps_mock = init_gapps_mock(application_id="my_app_id", +# process_status=ProcessStatusEnum.STARTING.value) +# log = Logger(gapps_mock) - log.debug("A message") +# log.debug("A message") - topic, message = gapps_mock.send.call_args.args +# topic, message = gapps_mock.send.call_args.args - assert expected_topic == topic - assert message['processStatus'] == ProcessStatusEnum.STARTING.value - assert message['logMessage'] == 'A message' +# assert expected_topic == topic +# assert message['processStatus'] == ProcessStatusEnum.STARTING.value +# assert message['logMessage'] == 'A message' -def test_platform_log(): +# def test_platform_log(): - application_id = "my_app" - gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) - log = Logger(gapps_mock) +# application_id = "my_app" +# gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) +# log = Logger(gapps_mock) - log.debug("foo") - gapps_mock.send.assert_called_once() +# log.debug("foo") +# gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args - assert message['source'] == application_id - assert message['logLevel'] == 'DEBUG' - assert message['logMessage'] == 'foo' - assert message['processStatus'] == ProcessStatusEnum.STOPPING.value - gapps_mock.send.reset_mock() +# assert message['source'] == application_id +# assert message['logLevel'] == 'DEBUG' +# assert message['logMessage'] == 'foo' +# assert message['processStatus'] == ProcessStatusEnum.STOPPING.value +# gapps_mock.send.reset_mock() - log.info("bar") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'INFO' == message['logLevel'] - assert 'bar' == message['logMessage'] - gapps_mock.send.reset_mock() +# log.info("bar") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'INFO' == message['logLevel'] +# assert 'bar' == message['logMessage'] +# gapps_mock.send.reset_mock() - log.error("bim") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'ERROR' == message['logLevel'] - assert 'bim' == message['logMessage'] - gapps_mock.send.reset_mock() +# log.error("bim") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'ERROR' == message['logLevel'] +# assert 'bim' == message['logMessage'] +# gapps_mock.send.reset_mock() - log.warning("baf") - gapps_mock.send.assert_called_once() - # send should have been passed a topic and a message - topic, message = gapps_mock.send.call_args.args - assert application_id == message['source'] - assert 'WARN' == message['logLevel'] - assert 'baf' == message['logMessage'] +# log.warning("baf") +# gapps_mock.send.assert_called_once() +# # send should have been passed a topic and a message +# topic, message = gapps_mock.send.call_args.args +# assert application_id == message['source'] +# assert 'WARN' == message['logLevel'] +# assert 'baf' == message['logMessage'] -def test_invalid_log_level(): - application_id = "my_app" - gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) - log = Logger(gapps_mock) +# def test_invalid_log_level(): +# application_id = "my_app" +# gapps_mock = init_gapps_mock(application_id=application_id, process_status=ProcessStatusEnum.STOPPING.value) +# log = Logger(gapps_mock) - with pytest.raises(AttributeError): - log.log("junk error", "BART") +# with pytest.raises(AttributeError): +# log.log("junk error", "BART") -def test_topic_and_status_set_correctly(): +# def test_topic_and_status_set_correctly(): - sim_id = "543" - application_id = "wicked_good_app_id" - mock_gapps = init_gapps_mock(simulation_id=sim_id, application_id=application_id, - process_status=ProcessStatusEnum.RUNNING.value) +# sim_id = "543" +# application_id = "wicked_good_app_id" +# mock_gapps = init_gapps_mock(simulation_id=sim_id, application_id=application_id, +# process_status=ProcessStatusEnum.RUNNING.value) - expected_topic = t.simulation_log_topic(sim_id) +# expected_topic = t.simulation_log_topic(sim_id) - log = Logger(mock_gapps) +# log = Logger(mock_gapps) - log.debug("A message") +# log.debug("A message") - # During the call to debug we expect the send function to be - # called on the mock object. Grab the arguments and then - # make sure that they are what we expect. - topic, message = mock_gapps.send.call_args.args +# # During the call to debug we expect the send function to be +# # called on the mock object. Grab the arguments and then +# # make sure that they are what we expect. +# topic, message = mock_gapps.send.call_args.args - assert message['source'] == application_id - assert topic == expected_topic - assert message['processStatus'] == "RUNNING" +# assert message['source'] == application_id +# assert topic == expected_topic +# assert message['processStatus'] == "RUNNING" diff --git a/gridappsd-python-lib/tests/test_logging_integration.py b/gridappsd-python-lib/tests/test_logging_integration.py index e4aa613..adb1e2c 100644 --- a/gridappsd-python-lib/tests/test_logging_integration.py +++ b/gridappsd-python-lib/tests/test_logging_integration.py @@ -1,97 +1,97 @@ -import os -import time +# import os +# import time -import mock -import pytest +# import mock +# import pytest -from gridappsd import GridAPPSD, topics as t -from gridappsd.loghandler import Logger +# from gridappsd import GridAPPSD, topics as t +# from gridappsd.loghandler import Logger -@pytest.fixture -def logger_and_gridapspd(gridappsd_client) -> (Logger, GridAPPSD): +# @pytest.fixture +# def logger_and_gridapspd(gridappsd_client) -> (Logger, GridAPPSD): - logger = Logger(gridappsd_client) +# logger = Logger(gridappsd_client) - yield logger, gridappsd_client +# yield logger, gridappsd_client - logger = None +# logger = None -@mock.patch.dict(os.environ, - dict(GRIDAPPSD_APPLICATION_ID='sample_app', - GRIDAPPSD_APPLICATION_STATUS='RUNNING')) -def test_log_stored(logger_and_gridapspd): - logger, gapps = logger_and_gridapspd +# @mock.patch.dict(os.environ, +# dict(GRIDAPPSD_APPLICATION_ID='sample_app', +# GRIDAPPSD_APPLICATION_STATUS='RUNNING')) +# def test_log_stored(logger_and_gridapspd): +# logger, gapps = logger_and_gridapspd - log_data_map = [ - (logger.debug, "A debug message", "DEBUG"), - (logger.info, "An info message", "INFO"), - (logger.error, "An error message", "ERROR"), - (logger.error, "Another error message", "ERROR"), - (logger.info, "Another info message", "INFO"), - (logger.debug, "A debug message", "DEBUG") - ] +# log_data_map = [ +# (logger.debug, "A debug message", "DEBUG"), +# (logger.info, "An info message", "INFO"), +# (logger.error, "An error message", "ERROR"), +# (logger.error, "Another error message", "ERROR"), +# (logger.info, "Another info message", "INFO"), +# (logger.debug, "A debug message", "DEBUG") +# ] - assert gapps.connected +# assert gapps.connected - # Make the calls to debug - for d in log_data_map: - d[0](d[1]) +# # Make the calls to debug +# for d in log_data_map: +# d[0](d[1]) - payload = { - "query": "select * from log order by timestamp" - } - time.sleep(5) - response = gapps.get_response(t.LOGS, payload, timeout=60) - assert response['data'], "There were not any records returned." +# payload = { +# "query": "select * from log order by timestamp" +# } +# time.sleep(5) +# response = gapps.get_response(t.LOGS, payload, timeout=60) +# assert response['data'], "There were not any records returned." - for x in response['data']: - if x['source'] != 'sample_app': - continue - expected = log_data_map.pop(0) - assert expected[1] == x['log_message'] - assert expected[2] == x['log_level'] +# for x in response['data']: +# if x['source'] != 'sample_app': +# continue +# expected = log_data_map.pop(0) +# assert expected[1] == x['log_message'] +# assert expected[2] == x['log_level'] -SIMULATION_ID='54321' +# SIMULATION_ID='54321' - #TODO Ask about loging api for simulations. -@mock.patch.dict(os.environ, - dict(GRIDAPPSD_APPLICATION_ID='new_sample_app', - GRIDAPPSD_APPLICATION_STATUS='RUNNING', - GRIDAPPSD_SIMULATION_ID=SIMULATION_ID)) -def test_simulation_log_stored(logger_and_gridapspd): - logger, gapps = logger_and_gridapspd +# #TODO Ask about loging api for simulations. +# @mock.patch.dict(os.environ, +# dict(GRIDAPPSD_APPLICATION_ID='new_sample_app', +# GRIDAPPSD_APPLICATION_STATUS='RUNNING', +# GRIDAPPSD_SIMULATION_ID=SIMULATION_ID)) +# def test_simulation_log_stored(logger_and_gridapspd): +# logger, gapps = logger_and_gridapspd - assert gapps.get_simulation_id() == SIMULATION_ID +# assert gapps.get_simulation_id() == SIMULATION_ID - log_data_map = [ - (logger.debug, "A debug message", "DEBUG"), - (logger.info, "An info message", "INFO"), - (logger.error, "An error message", "ERROR"), - (logger.error, "Another error message", "ERROR"), - (logger.info, "Another info message", "INFO"), - (logger.debug, "A debug message", "DEBUG") - ] +# log_data_map = [ +# (logger.debug, "A debug message", "DEBUG"), +# (logger.info, "An info message", "INFO"), +# (logger.error, "An error message", "ERROR"), +# (logger.error, "Another error message", "ERROR"), +# (logger.info, "Another info message", "INFO"), +# (logger.debug, "A debug message", "DEBUG") +# ] - assert gapps.connected +# assert gapps.connected - # Make the calls to debug - for d in log_data_map: - d[0](d[1]) +# # Make the calls to debug +# for d in log_data_map: +# d[0](d[1]) - time.sleep(5) - payload = { - "query": "select * from log" - } +# time.sleep(5) +# payload = { +# "query": "select * from log" +# } - response = gapps.get_response(t.LOGS, payload, timeout=60) - assert response['data'], "There were not any records returned." +# response = gapps.get_response(t.LOGS, payload, timeout=60) +# assert response['data'], "There were not any records returned." - for x in response['data']: - if x['source'] != 'new_sample_app': - continue - expected = log_data_map.pop(0) - assert expected[1] == x['log_message'] - assert expected[2] == x['log_level'] +# for x in response['data']: +# if x['source'] != 'new_sample_app': +# continue +# expected = log_data_map.pop(0) +# assert expected[1] == x['log_message'] +# assert expected[2] == x['log_level'] diff --git a/gridappsd-python-lib/tests/test_simulation.py b/gridappsd-python-lib/tests/test_simulation.py index b4c85d2..5be2be8 100644 --- a/gridappsd-python-lib/tests/test_simulation.py +++ b/gridappsd-python-lib/tests/test_simulation.py @@ -1,45 +1,45 @@ -import json -# from pprint import pprint -import logging -import os -import sys -import time -import pytest +# import json +# # from pprint import pprint +# import logging +# import os +# import sys +# import time +# import pytest -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) +# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -from gridappsd import GridAPPSD, topics as t -from gridappsd.simulation import Simulation +# from gridappsd import GridAPPSD, topics as t +# from gridappsd.simulation import Simulation -# The directory containing this file -HERE = os.path.dirname(__file__) +# # The directory containing this file +# HERE = os.path.dirname(__file__) -def base_config(): - data = {"power_system_config":{"SubGeographicalRegion_name":"_ABEB635F-729D-24BF-B8A4-E2EF268D8B9E","GeographicalRegion_name":"_73C512BD-7249-4F50-50DA-D93849B89C43","Line_name":"_49AD8E07-3BF9-A4E2-CB8F-C3722F837B62"},"simulation_config":{"power_flow_solver_method":"NR","duration":120,"simulation_name":"ieee13nodeckt","simulator":"GridLAB-D","start_time":1605418946,"run_realtime":False,"simulation_output":{},"model_creation_config":{"load_scaling_factor":1.0,"triplex":"y","encoding":"u","system_frequency":60,"voltage_multiplier":1.0,"power_unit_conversion":1.0,"unique_names":"y","schedule_name":"ieeezipload","z_fraction":0.0,"i_fraction":1.0,"p_fraction":0.0,"randomize_zipload_fractions":False,"use_houses":False},"simulation_broker_port":51044,"simulation_broker_location":"127.0.0.1"},"application_config":{"applications":[]},"service_configs":[],"test_config":{"randomNum":{"seed":{"value":185213303967438},"nextNextGaussian":0.0,"haveNextNextGaussian":False},"events":[],"testInput":True,"testOutput":True,"appId":"","testId":"1468836560","testType":"simulation_vs_expected","storeMatches":False},"simulation_request_type":"NEW"} - # with open("{HERE}/simulation_fixtures/13_node_2_min_base.json".format(HERE=HERE)) as fp: - # data = json.load(fp) - return data +# def base_config(): +# data = {"power_system_config":{"SubGeographicalRegion_name":"_ABEB635F-729D-24BF-B8A4-E2EF268D8B9E","GeographicalRegion_name":"_73C512BD-7249-4F50-50DA-D93849B89C43","Line_name":"_49AD8E07-3BF9-A4E2-CB8F-C3722F837B62"},"simulation_config":{"power_flow_solver_method":"NR","duration":120,"simulation_name":"ieee13nodeckt","simulator":"GridLAB-D","start_time":1605418946,"run_realtime":False,"simulation_output":{},"model_creation_config":{"load_scaling_factor":1.0,"triplex":"y","encoding":"u","system_frequency":60,"voltage_multiplier":1.0,"power_unit_conversion":1.0,"unique_names":"y","schedule_name":"ieeezipload","z_fraction":0.0,"i_fraction":1.0,"p_fraction":0.0,"randomize_zipload_fractions":False,"use_houses":False},"simulation_broker_port":51044,"simulation_broker_location":"127.0.0.1"},"application_config":{"applications":[]},"service_configs":[],"test_config":{"randomNum":{"seed":{"value":185213303967438},"nextNextGaussian":0.0,"haveNextNextGaussian":False},"events":[],"testInput":True,"testOutput":True,"appId":"","testId":"1468836560","testType":"simulation_vs_expected","storeMatches":False},"simulation_request_type":"NEW"} +# # with open("{HERE}/simulation_fixtures/13_node_2_min_base.json".format(HERE=HERE)) as fp: +# # data = json.load(fp) +# return data -def test_simulation_no_duplicate_measurement_timestamps(gridappsd_client: GridAPPSD): - num_measurements = 0 - timestamps = set() +# def test_simulation_no_duplicate_measurement_timestamps(gridappsd_client: GridAPPSD): +# num_measurements = 0 +# timestamps = set() - def measurement(sim, timestamp, measurement): - nonlocal num_measurements - num_measurements += 1 - assert timestamp not in timestamps - timestamps.add(timestamp) +# def measurement(sim, timestamp, measurement): +# nonlocal num_measurements +# num_measurements += 1 +# assert timestamp not in timestamps +# timestamps.add(timestamp) - gapps = gridappsd_client - sim = Simulation(gapps, base_config()) - sim.add_onmeasurement_callback(measurement) - sim.start_simulation() - sim.run_loop() +# gapps = gridappsd_client +# sim = Simulation(gapps, base_config()) +# sim.add_onmeasurement_callback(measurement) +# sim.start_simulation() +# sim.run_loop() - # did we get a measurement? - assert num_measurements > 0 +# # did we get a measurement? +# assert num_measurements > 0 - # if empty then we know the simulation did not work. - assert timestamps +# # if empty then we know the simulation did not work. +# assert timestamps From 62182be6acfd36a1c467d1075f198f4a03a88724 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:27:09 -0700 Subject: [PATCH 11/33] Update to use scripts for main processes. --- .github/workflows/deploy-dev-release.yml | 250 ++- .../gridappsd/__no_init__here | 0 gridappsd-field-bus-lib/info/CHANGELOG.md | 0 gridappsd-field-bus-lib/info/VERSION | 1 + gridappsd-field-bus-lib/pyproject.toml | 10 +- gridappsd-python-lib/CHANGELOG.md | 0 .../gridappsd/docker_handler.py | 1373 +++++++++-------- gridappsd-python-lib/info/CHANGELOG.md | 0 gridappsd-python-lib/info/VERSION | 1 + gridappsd-python-lib/pyproject.toml | 7 +- info/CHANGELOG.md | 0 info/VERSION | 1 + pyproject.toml | 4 +- scripts/create_local_version.sh | 30 + scripts/poetry_build.sh | 57 + scripts/poetry_install.sh | 21 + scripts/poetry_update.sh | 21 + scripts/projects.sh | 4 + scripts/replace_path_deps.sh | 63 + scripts/run_on_each.sh | 18 + 20 files changed, 1187 insertions(+), 674 deletions(-) create mode 100644 gridappsd-field-bus-lib/gridappsd/__no_init__here create mode 100644 gridappsd-field-bus-lib/info/CHANGELOG.md create mode 100644 gridappsd-field-bus-lib/info/VERSION create mode 100644 gridappsd-python-lib/CHANGELOG.md create mode 100644 gridappsd-python-lib/info/CHANGELOG.md create mode 100644 gridappsd-python-lib/info/VERSION create mode 100644 info/CHANGELOG.md create mode 100644 info/VERSION create mode 100755 scripts/create_local_version.sh create mode 100755 scripts/poetry_build.sh create mode 100755 scripts/poetry_install.sh create mode 100755 scripts/poetry_update.sh create mode 100644 scripts/projects.sh create mode 100755 scripts/replace_path_deps.sh create mode 100755 scripts/run_on_each.sh diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 9d1a448..20679a8 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -1,12 +1,17 @@ ---- -name: Deploy Pre-Release Artifacts +name: Deploy Release Artifacts on: push: branches: - - develop + - develop workflow_dispatch: - + inputs: + release-version: + description: "Version number to use. If provided bump-rule will be ignored" + required: false + default: "" + type: string + defaults: run: shell: bash @@ -14,16 +19,237 @@ defaults: env: LANG: en_US.utf-8 LC_ALL: en_US.utf-8 - PYTHON_VERSION: '3.10' + PYTHON_VERSION: "3.10" jobs: - call-deploy-release: + deploy-dev-release: + runs-on: ubuntu-22.04 permissions: - contents: write # To push a branch - pull-requests: write # To create a PR from that branch + contents: write # To push a branch + pull-requests: write # To create a PR from that branch + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: develop + token: ${{ secrets.git-token }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1.3.3 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + # #---------------------------------------------- + # # load cached venv if cache exists + # #---------------------------------------------- + # - name: Load cached venv + # id: cached-poetry-dependencies + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + # #---------------------------------------------- + # # install dependencies if cache does not exist + # #---------------------------------------------- + # - name: Install dependencies + # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # run: poetry install --no-interaction --no-root + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: | + ./scripts/poetry_install.sh + + # git checkout develop + # poetry lock --no-update + # poetry install --no-interaction + + # - name: Use given release-version number + # if: inputs.release-version != '' + # run: | + # echo "Using given release version is ${{ inputs.release-version }}" + # poetry version ${{ inputs.release-version }} + + # NEW_TAG=v$(poetry version --short) + + # # we want to be able to use the variable in later + # # steps we set a NEW_TAG environmental variable + # echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + # # we don't want to update pyproject.toml yet. don't want this change to create merge conflict. + # # we don't really persist right version in pyproject.toml to figure out the next version. we use git tags. + # git restore pyproject.toml + + #---------------------------------------------- + # bump version number for patch + #---------------------------------------------- + - name: Bump Version + run: | + # current_tag is the last tagged release in the repository. From there + # we need to remove the v from the beginning of the tag. + # echo "Bump rule is ${{ inputs.bump-rule }}" + # echo "Given release version is ${{ inputs.release-version }}" + dt=$(date +%Y.%-m.0) + if ! $(git tag -l "v*" = ''); then + # uses -V which is version sort to keep it monotonically increasing. + current_tag=$(git tag -l "v*" | grep --invert-match '-' | sort --reverse -V | sed -n 1p) + echo "current git tag is ${current_tag}" + current_tag=${current_tag#?} + if [[ "$current_tag" < "$dt" ]]; then + current_tag=$dt + fi + # current_tag is now the version we want to set our poetry version so + # that we can bump the version + ./scripts/run_on_each.sh poetry version ${current_tag} + + # poetry version ${current_tag} + # poetry version prerelease --no-interaction + + else + # very first release. start with inputs.release-version + + echo "First release. Setting tag as 0.1.0rc0" + current_tag=$(date +%Y.%-m.1) + ./scripts/run_on_each.sh poetry version ${current_tag} + + # poetry version ${current_tag} + fi + + NEW_TAG=v$(poetry version --short) + + # Finally because we want to be able to use the variable in later + # steps we set a NEW_TAG environmental variable + echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + # we don't want to update pyproject.toml yet. don't want this change to create merge conflict. + # we don't really persist right version in pyproject.toml to figure out the next version. we use git tags. + #git restore pyproject.toml + + # #-------------------------------------------------------------- + # # Create a new releases/new_tag + # #-------------------------------------------------------------- + # - name: Create a new releases branch + # run: | + # git checkout -b releases/${NEW_TAG} + # git push --set-upstream origin releases/${NEW_TAG} + + # #-------------------------------------------------------------- + # # merge changes back to main + # #-------------------------------------------------------------- + # - name: Merge changes back to main + # run: | + # git checkout main + # git merge ${{ inputs.merge-strategy }} releases/${NEW_TAG} + # git push + + # - name: Run tests on main branch + # id: run-tests-on-main + # run: | + # poetry run pytest --timeout=${{ inputs.run-tests-wait }} tests + # continue-on-error: true + + # - name: Do something with a failing build + # if: steps.run-tests-on-main.outcome != 'success' + # run: | + # echo "tests on main did not succeed. Outcome is ${{ steps.run-tests-on-main.outcome }}" + # git reset --hard HEAD~1 + # git push origin HEAD --force + # git branch -d releases/${NEW_TAG} + # git push origin --delete releases/${NEW_TAG} + # echo "reverted changes to main and removed release branch" + # exit 1 + + - name: Create build artifacts + run: | + # set the right version in pyproject.toml before build and publish + ./scripts/poetry_build.sh + ./scripts/replace_path_deps.sh + + set -x + set -u + set -e + + # all python packages, in topological order + . scripts/projects.sh + _projects=". ${PROJECTS}" + echo "Running on following projects: ${_projects}" + for p in $_projects + do + cd "${p}" || exit + echo "==running in ${p}==" + ../scripts/replace_path_deps.sh + done + + # poetry version ${NEW_TAG#?} + # poetry build -vvv + + # - uses: ncipollo/release-action@v1 + # with: + # artifacts: "dist/*.gz,dist/*.whl" + # artifactErrorsFailBuild: true + # generateReleaseNotes: true + # commit: ${{ github.ref }} + # # check bump-rule and set accordingly + # prerelease: true + # tag: ${{ env.NEW_TAG }} + # token: ${{ secrets.git-token }} + + # - name: Publish to pypi + # id: publish-to-pypi + # if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' + # run: | + # echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV + # ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.pypi-token }} + # ./scripts/run_on_each.sh poetry publish + + # - name: Publish to test-pypi + # id: publish-to-test-pypi + # if: ${{ inputs.publish-to-test-pypi }} + # run: | + # poetry config repositories.test-pypi https://test.pypi.org/legacy/ + # poetry config pypi-token.test-pypi ${{ secrets.pypi-token }} + # poetry publish -r test-pypi + # continue-on-error: true + + # - name: if publish to pypi/test-pypi failed revert main and delete release branch + # if: ${{ steps.publish-to-pypi.outcome != 'success' && steps.publish-to-test-pypi.outcome != 'success' }} + # run: | + # echo "publish to pypi/test-pypi did not succeed. Outcome for pypi = ${{ steps.publish-to-pypi.outcome }} outcome for test-pypi= ${{ steps.publish-to-test-pypi.outcome }}" + # git reset --hard HEAD~1 + # git push origin HEAD --force + # git branch -d releases/${NEW_TAG} + # git push origin --delete releases/${NEW_TAG} + # echo "reverted changes to main and removed release branch" - uses: GRIDAPPSD/.github/.github/workflows/deploy-dev-release.yml@main - secrets: - git-token: ${{ secrets.GITHUB_TOKEN }} - pypi-token: ${{ secrets.PYPI_TOKEN }} + # - name: if publish to pypi/test-pypi failed delete release and tag on github + # if: ${{ ! (steps.publish-to-pypi.outcome == 'success' || steps.publish-to-test-pypi.outcome == 'success') }} + # uses: dev-drprasad/delete-tag-and-release@v0.2.1 + # env: + # GITHUB_TOKEN: ${{ secrets.git-token }} + # with: + # tag_name: ${{ env.NEW_TAG }} + # - name: if publish to pypi/test-pypi failed exit with exit code 1 + # if: ${{ steps.publish-to-pypi.outcome != 'success' && steps.publish-to-test-pypi.outcome != 'success' }} + # run: | + # exit 1 diff --git a/gridappsd-field-bus-lib/gridappsd/__no_init__here b/gridappsd-field-bus-lib/gridappsd/__no_init__here new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/info/CHANGELOG.md b/gridappsd-field-bus-lib/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-field-bus-lib/info/VERSION b/gridappsd-field-bus-lib/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/gridappsd-field-bus-lib/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index c01cb53..0c9ecd6 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-field-bus" -version = "v2.8.230413" +version = "2023.5.2a0" description = "GridAPPS-D Field Bus Implementation" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -21,16 +21,10 @@ packages = [ { include = 'gridappsd'} ] -[tool.poetry.scripts] -# Add things in the form -# myscript = 'my_package:main' -register_app = 'gridappsd.register_app:main' -gridappsd-cli = 'gridappsd.cli:_main' - [tool.poetry.dependencies] python = ">=3.7.9,<4.0" -gridappsd-python = {path="../gridappsd-python-lib", develop=true} +gridappsd-python = "^2023.5.2a0" cim-graph = "^0.1.20230413185916a0" [tool.poetry.group.dev.dependencies] diff --git a/gridappsd-python-lib/CHANGELOG.md b/gridappsd-python-lib/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-python-lib/gridappsd/docker_handler.py b/gridappsd-python-lib/gridappsd/docker_handler.py index 5b3776b..29bf751 100644 --- a/gridappsd-python-lib/gridappsd/docker_handler.py +++ b/gridappsd-python-lib/gridappsd/docker_handler.py @@ -1,650 +1,723 @@ -#!/usr/bin/env python3 - -import contextlib -import logging -import os -import random -import re -import shutil -import tarfile -import threading -import urllib.request -from copy import deepcopy -from datetime import datetime, timedelta -from pathlib import Path -from pprint import pformat -from subprocess import PIPE -from typing import Optional, Union - -import stomp -import time -from docker.models.containers import Container - -from gridappsd import GridAPPSD -from gridappsd.goss import GRIDAPPSD_ENV_ENUM - -logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) -logging.getLogger("docker.auth").setLevel(logging.INFO) -logging.getLogger("docker.utils").setLevel(logging.INFO) - -_log = logging.getLogger("gridappsd.docker_handler") - - -try: - import docker - HAS_DOCKER = True -except ImportError: - _log.warning("Docker api not loaded. pip install docker to install as package.") - HAS_DOCKER = False - -if HAS_DOCKER: - - # The following variable is used for creating a volume for the gridappsd container - # to utilize. It allows the ability to use multiple containers to run tests - # along side each other. - GRIDAPPSD_CONFIG_VOLUME_NAME = f"gridappsd_config_{random.randint(1,100)}" - - # This named container will be used to hold configuration/folders so that other containers - # that start can use them. To use add "volume_from": [CONFIGURATION_CONTAINER_NAME] and - # the mount point within the container will also be within the service container. - CONFIGURATION_CONTAINER_NAME = "testconfig" - - def expand_all(user_path): - return os.path.expandvars(os.path.expanduser(user_path)) - - __TMP_ROOT__ = "/tmp/assets" - if Path(__TMP_ROOT__).exists(): - shutil.rmtree(__TMP_ROOT__, ignore_errors=True) - os.makedirs(__TMP_ROOT__) - - # This path needs to be the path to the repo where configuration files are located. - GRIDAPPSD_CONF_DIR = Path(__file__).resolve().parent.parent.joinpath("gridappsd/conf") - - assert Path(GRIDAPPSD_CONF_DIR).joinpath("entrypoint.sh").exists() - assert Path(GRIDAPPSD_CONF_DIR).joinpath("run-gridappsd.sh").exists() - - GRIDAPPSD_DATA_REPO = str(Path(__TMP_ROOT__).joinpath("mysql").resolve()) - - os.makedirs(GRIDAPPSD_DATA_REPO, exist_ok=True) - if not Path(GRIDAPPSD_DATA_REPO).is_dir(): - raise AttributeError(f"Invalid GRIDAPPSD_DATA_REPO couldn't make or doesn't exist {GRIDAPPSD_DATA_REPO}") - - MYSQL_SCHEMA_INIT_DIR = f'{GRIDAPPSD_DATA_REPO}/docker-entrypoint-initdb.d' - - - def mysql_setup(): - """ - Downloads gridappsd_mysql_dump.sql into the MYSQL_SCHEMA_INIT_DIR init directory. - This will then be mounted into the mysql container. - """ - # Downlaod mysql file - _log.debug("Downloading mysql data file from Bootstrap repository") - mysql_file = f'{MYSQL_SCHEMA_INIT_DIR}/gridappsd_mysql_dump.sql' - if os.path.isdir(mysql_file): - raise RuntimeError(f"mysql datafile is directory, delete {mysql_file} using sudo rm -rf {mysql_file}") - if not os.path.isdir(MYSQL_SCHEMA_INIT_DIR): - os.makedirs(MYSQL_SCHEMA_INIT_DIR, 0o0775, exist_ok=True) - urllib.request.urlretrieve( - 'https://raw.githubusercontent.com/GRIDAPPSD/Bootstrap/master/gridappsd_mysql_dump.sql', - filename=mysql_file) - - # Modify the mysql file to allow connections from gridappsd container - with open(mysql_file, "r") as sources: - lines = sources.readlines() - with open(mysql_file, "w") as sources: - for line in lines: - sources.write(re.sub(r'localhost', '%', line)) - assert Path(mysql_file).exists() - - # Use the environmental variable if specified otherwise use the develop tag. - DEFAULT_GRIDAPPSD_TAG = os.environ.get('GRIDAPPSD_TAG_ENV', 'develop') - - # Network to connect all of the containers up to by default. - NETWORK = "test_my_network" - - __TPL_DEPENDENCY_CONFIG__ = { - "influxdb": { - "start": True, - "image": "gridappsd/influxdb:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8086/tcp": 8086}, - "environment": {"INFLUXDB_DB": "proven"} - }, - "redis": { - "start": True, - "image": "redis:3.2.11-alpine", - "pull": True, - "ports": {"6379/tcp": 6379}, - "environment": [], - "entrypoint": "redis-server --appendonly yes", - }, - "blazegraph": { - "start": True, - "image": "gridappsd/blazegraph:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8080/tcp": 8889}, - "environment": [] - }, - "mysql": { - "start": True, - "image": "mysql/mysql-server:5.7", - "pull": True, - "ports": {"3306/tcp": 3306}, - "environment": { - "MYSQL_RANDOM_ROOT_PASSWORD": "yes", - "MYSQL_PORT": "3306" - }, - # "volumes": { - # "/home/gridappsd/test-assets": {"bind": "/whatthehell", "mode": "rw"} - # #"test-assets": {"bind": "/whatthehell", "mode": "rw"} - # #data_dir + "/dumps/": {"bind": "/docker-entrypoint-initdb.d/", "mode": "ro"} - # }, - # Our own so we can create - "volumes_required": [ - dict(name="mysql_config", - local_path=MYSQL_SCHEMA_INIT_DIR, - container_path="/docker-entrypoint-initdb.d") - ], - "volumes_from": [ - "mysql_config" - ], - "onsetupfn": mysql_setup - }, - "proven": { - "start": True, - "image": "gridappsd/proven:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"8080/tcp": 18080}, - "environment": { - "PROVEN_SERVICES_PORT": "18080", - "PROVEN_SWAGGER_HOST_PORT": "localhost:18080", - "PROVEN_USE_IDB": "true", - "PROVEN_IDB_URL": "http://influxdb:8086", - "PROVEN_IDB_DB": "proven", - "PROVEN_IDB_RP": "autogen", - "PROVEN_IDB_USERNAME": "root", - "PROVEN_IDB_PASSWORD": "root", - "PROVEN_T3DIR": "/proven"}, - "links": {"influxdb": "influxdb"} - } - } - - __TPL_GRIDAPPSD_CONFIG__ = { - "gridappsd": { - "start": True, - "image": "gridappsd/gridappsd:{{DEFAULT_GRIDAPPSD_TAG}}", - "pull": True, - "ports": {"61613/tcp": 61613, "61614/tcp": 61614, "61616/tcp": 61616}, - "environment": { - "PATH": "/gridappsd/bin:/gridappsd/lib:/gridappsd/services/fncsgossbridge/service:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin", - "DEBUG": 1, - "START": 1 - }, - "links": {"mysql": "mysql", - "influxdb": "influxdb", - "blazegraph": "blazegraph", - "proven": "proven", - "redis": "redis"}, - "volumes_required": [ - dict(name=GRIDAPPSD_CONFIG_VOLUME_NAME, - local_path=GRIDAPPSD_CONF_DIR, - container_path="/startup/conf") - ], - "volumes_from": [ - GRIDAPPSD_CONFIG_VOLUME_NAME - ], - "entrypoint": "/startup/conf/entrypoint.sh", - "command": "/startup/conf/entrypoint.sh" - } - } - - def __update_template_data__(data, update_dict): - data_cpy = deepcopy(data) - for k, v in data_cpy.items(): - for u, p in update_dict.items(): - v['image'] = v['image'].replace(u, p) - - return data_cpy - - - __replace_dict__ = {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG} - DEFAULT_DOCKER_DEPENDENCY_CONFIG = __update_template_data__(__TPL_DEPENDENCY_CONFIG__, __replace_dict__) - DEFAULT_GRIDAPPSD_DOCKER_CONFIG = __update_template_data__(__TPL_GRIDAPPSD_CONFIG__, __replace_dict__) - - def update_gridappsd_tag(new_gridappsd_tag): - """ - Update the default tag used within the dependency and gridappsd containers to be - what is specified in the new gridappsd_tag variable - """ - global DEFAULT_GRIDAPPSD_TAG - - DEFAULT_GRIDAPPSD_TAG = new_gridappsd_tag - _log.info(f"Updated gridappsd docker tag {DEFAULT_GRIDAPPSD_TAG} ") - __replace_dict__.update({"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG}) - DEFAULT_DOCKER_DEPENDENCY_CONFIG.update(__update_template_data__(__TPL_DEPENDENCY_CONFIG__, __replace_dict__)) - DEFAULT_GRIDAPPSD_DOCKER_CONFIG.update(__update_template_data__(__TPL_GRIDAPPSD_CONFIG__, __replace_dict__)) - - - class Containers: - """ - This class allows the creation/management of containers created by the gridappsd - docker process. - """ - __client__ = docker.from_env() - - def __init__(self, container_def): - self._container_def = container_def - - @property - def container_def(self): - return self._container_def - - @staticmethod - def remove_container(name): - try: - container = Containers.__client__.containers.get(name) - container.kill() - except docker.errors.NotFound: - _log.debug(f"container {name} not found so couldn't remove.") - - @staticmethod - def container_list(ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): - """ - Provides a wrapper around the listing of docker containers. This function was - brought about when running from within a docker container. - - Currently the docker container that is run using docker-compose up from the - gridappsd-dev-environment is specified as gridappsd_dev, however this can change - and could potentially be extended to multiple named containers. - - @param: ignore_list: - optional container names to not stop :: A string or list of strings - """ - - if ignore_list is None: - ignore_list = [] - elif isinstance(ignore_list, str): - ignore_list = [ignore_list] - containers = [] - for cont in Containers.__client__.containers.list(): - if cont.name not in ignore_list: - containers.append(cont) - return containers - - @staticmethod - def reset_all_containers(ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): - """ - Provides a wrapper around the resetting of all docker containers. This function was - brought about when running from within a docker container. - - Currently the docker container that is run using docker-compose up from the - gridappsd-dev-environment is specified as gridappsd_dev, however this can change - and could potentially be extended to multiple named containers. - - @param: ignore_list: - optional container names to not stop :: A string or list of strings - """ - if ignore_list is None: - ignore_list = [] - elif isinstance(ignore_list, str): - ignore_list = [ignore_list] - - for cont in Containers.__client__.containers.list(): - if cont.name not in ignore_list: - cont.kill() - - @staticmethod - def check_required_running(self, config): - my_config = deepcopy(config) - - for c in Containers.__client__.containers.list(): - my_config.pop(c.name, None) - - assert not my_config, f"The required containers were not satisfied missing {list(my_config.keys())}" - - @staticmethod - def create_volume_container(name, volume_name, - mount_in_container_at, restart_if_exists: bool = False, - mode="rw"): - - _log.info(f"Creating container {name} and mounting volume {volume_name} " - f"at {mount_in_container_at} in container {name}") - client = docker.from_env() - - should_create = False - try: - cont = Containers.__client__.containers.get(name) - except docker.errors.NotFound: - # start container - should_create = True - else: - if restart_if_exists: - should_create = True - cont.stop() - else: - return cont - - if should_create: - kwargs = {} - kwargs['image'] = "alpine" - # Only name the containers if remove is on - kwargs['remove'] = True - kwargs['name'] = name - kwargs['detach'] = True - kwargs['volumes'] = { - volume_name: {"bind": mount_in_container_at, "mode": mode} - } - # keep the container running - kwargs['command'] = "tail -f /dev/null" - cont = client.containers.run(**kwargs) - _log.info(f"New container for volume created {cont.name}") - # print(f"New container is: {container.name}") - return cont - - @staticmethod - def create_get_network(name: str) -> docker.models.networks.Network: - try: - network = Containers.__client__.networks.get(name) - except docker.errors.NotFound: - network = Containers.__client__.networks.create(name, driver="bridge") - - return network - - @staticmethod - def copy_to(src: str, dst: str): - """ - Copy a local directory onto a destination - - .. note:: - - Make sure the dst (destination) is in the form container:directory - - @param: src: The local directory to copy from the host - @param: dst: The container:destination to copy the src to. - """ - _log.debug(f"copying folder from: {src} to {dst}") - src = str(src) - # python3.6 can't combine PosixPath and str - assert os.path.exists(src), f"{src} does not exist!" - client = docker.from_env() - name, dst = dst.split(':') - container = client.containers.get(name) - assert container is not None - cwd = os.getcwd() - os.chdir(os.path.dirname(src)) - srcname = os.path.basename(src) - tarfilename = src + '.tar' - tar = tarfile.open(tarfilename, mode='w') - try: - tar.add(srcname) - finally: - tar.close() - - data = open(tarfilename, 'rb').read() - resp = container.put_archive(os.path.dirname(dst), data) - os.chdir(cwd) - _log.debug(f"Response from put_archive {resp}") - - def start(self): - _log.info("Starting containers") - # Containers.create_volume_container(CONFIGURATION_CONTAINER_NAME, "testconfig", "/testconfig") - #pprint(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) - client = docker.from_env() - # print(f"Docker client version: {client.version()}") - for service, value in self._container_def.items(): - _log.info(f"Pulling {service} image") - _log.info(f"Pulling {service} : {self._container_def[service]['image']}") - client.images.pull(self._container_def[service]['image']) - try: - container = client.containers.get(service) - self._container_def[service]['containerid'] = container.id - except docker.errors.NotFound: - _log.debug(f"Couldn't find {service} so continuing on.") - - # Provide a way to dynamically create things that the container will need - # on the host system. This is important if we want to create a volume before - # starting the container up. - if value.get('onsetupfn'): - value.get('onsetupfn')() - - if self._container_def[service].get("volumes_required"): - for vr in self._container_def[service].get("volumes_required"): - _log.info(f"Creating volume for {service}: name={vr['name']}, " - f"volume_name={vr['name']}, container_path={vr['container_path']}") - Containers.create_volume_container( - name=vr["name"], - volume_name=vr["name"], - mount_in_container_at=vr["container_path"], - restart_if_exists=True - ) - time.sleep(20) - if vr.get("local_path"): - _log.debug(f"contents of local path({vr.get('local_path')}):\n\t{os.listdir(vr.get('local_path'))}") - _log.info(f"Copy to mounted volume for {service}: " - f"local_path={vr['local_path']}, container_path={vr['container_path']}") - Containers.copy_to(vr["local_path"], f'{vr["name"]}:{vr["container_path"]}') - network = Containers.create_get_network(NETWORK) - for service, value in self._container_def.items(): - if self._container_def[service]['start']: - _log.debug(f"Starting {service} : {self._container_def[service]['image']}") - kwargs = {} - kwargs['image'] = self._container_def[service]['image'] - # Only name the containers if remove is on - kwargs['remove'] = True - kwargs['name'] = service - kwargs['detach'] = True - kwargs['entrypoint'] = value.get('entrypoint') - if self._container_def[service].get('entrypoint'): - kwargs['entrypoint'] = value['entrypoint'] - if self._container_def[service].get('environment'): - kwargs['environment'] = value['environment'] - if self._container_def[service].get('ports'): - kwargs['ports'] = value['ports'] - if self._container_def[service].get('volumes'): - kwargs['volumes'] = value['volumes'] - if self._container_def[service].get('entrypoint'): - kwargs['entrypoint'] = value['entrypoint'] - # if self._container_def[service].get('links'): - # kwargs['links'] = value['links'] - if self._container_def[service].get('volumes_from'): - kwargs['volumes_from'] = value['volumes_from'] - #for k, v in kwargs.items(): - # print(f"k->{k}, v->{v}") - _log.debug("Starting container with the following args:") - _log.debug(f"{pformat(kwargs)}") - launched_container = None - try: - container = client.containers.get(service) - _log.debug("Found existing container") - except docker.errors.NotFound: - container = client.containers.run(**kwargs) - _log.debug("Started new container") - network.connect(container.id) - self._container_def[service]['containerid'] = container.id - _log.debug(f"Current containers are: {[x.name for x in client.containers.list()]}") - - def wait_for_log_pattern(self, container, pattern, timeout=60): - assert self._container_def.get(container), f"Container {container} is not in definition." - client = docker.from_env() - container = client.containers.get(self._container_def.get(container)['containerid']) - until = datetime.now() + timedelta(seconds=timeout) - for p in container.logs(stream=True, until=until): - _log.info(f"HANDLER: {p.decode('utf-8')}") - if pattern in p.decode('utf-8'): - print(f"Found pattern {pattern}") - return - raise TimeoutError(f"Pattern {pattern} was not found in logs of container {container} within {timeout}s") - - def wait_for_http_ok(self, url, timeout=30): - import requests - results = None - test_count = 0 - - while results is None or not results.ok: - test_count += 1 - if test_count > timeout: - raise TimeoutError(f"Could not reach {url} within allotted timeout {timeout}s") - results = requests.get(url) - time.sleep(1) - - print(f"Found url {url} within timeout {timeout}") - - def stop(self): - client = docker.from_env() - for service, value in self._container_def.items(): - if value.get('containerid'): - try: - cnt = client.containers.get(value.get('containerid')) - cnt.kill() - time.sleep(2) - - if "volumes_required" in value: - # Loop over the volumes that are required for each image and - # remove the volume. - for volume_spec in value["volumes_required"]: - Containers.remove_container(volume_spec['name']) - time.sleep(2) - - # client.containers.get(value.get('containerid')).kill() # value.get('name')).kill() - except docker.errors.NotFound as ex: - _log.error(f"Volume {value.get('containerid')} was not found.") - _log.exception(ex) - - - threads = [] - - def stream_container_log_to_file(container_name: str, logfile: str): - client = docker.from_env() - container = client.containers.list(filters=dict(name=container_name))[0] - - print(container) - - def log_output(): - nonlocal container, logfile - print(f"Starting to write to file {logfile}.") - with open(logfile, 'wb') as fp: - print(f"openfile") - for p in container.logs(stream=True, stderr=PIPE, stdout=PIPE): - fp.write(p) - fp.flush() - - threading.Thread(target=log_output, daemon=True).start() - - - @contextlib.contextmanager - def run_containers(config, stop_after=True) -> Containers: - containers = Containers(config) - - containers.start() - try: - yield containers - finally: - if stop_after: - containers.stop() - - - @contextlib.contextmanager - def run_dependency_containers(stop_after=False) -> Containers: - - containers = Containers(DEFAULT_DOCKER_DEPENDENCY_CONFIG) - - containers.start() - try: - yield containers - finally: - if stop_after: - containers.stop() - - - @contextlib.contextmanager - def run_gridappsd_container(stop_after=True, rebuild_if_present=False) -> Container: - """ A contextmanager that uses """ - - parent_container = get_docker_in_docker() - - client = docker.from_env() - # if there is a parent_container then we need to make sure that it is connected - # to the same network as our systems. If not then we need to modify the network - # to include the parent container - if parent_container: - env = DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd'].get('environment') - if env is None: - env = {} - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.name] = 'gridappsd' - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.name] = 'test_app_user' - env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.name] = '4Test' - DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd']['environment'] = env - - _log.debug(f"Running inside a container environment: {parent_container.name}") - network = client.networks.get(NETWORK) - has_it = False - for x in network.containers: - if x.name == parent_container.name: - has_it = True - _log.debug(f"parent_container {parent_container.name} is connected to the network.") - break - if not has_it: - _log.debug(f"Connecting new container to the network: {parent_container.name}") - network.connect(parent_container) - else: - _log.debug("Not running in a container") - - containers = Containers(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) - - gridappsd_container = None - try: - gridappsd_container = client.containers.get("gridappsd") - _log.info(f"{gridappsd_container.name} container found") - if rebuild_if_present: - gridappsd_container.kill() - except docker.errors.NotFound: - _log.debug("gridappsd container not found!") - - try: - if gridappsd_container is None: - containers.start() - - # the gridappsd container itself will take a bit to start up. - time.sleep(30) - - tries = 30 - while True: - tries -= 1 - if tries <=0: - raise RuntimeError("Couldn't connect to gridappsd server in a timely manner!") - try: - g = GridAPPSD() - if g.connected: - _log.info("Connected to gridappsd!") - g.disconnect() - break - - except stomp.exception.ConnectFailedException or stomp.exception.NotConnectedException: - _log.error("Retesting connection") - - yield gridappsd_container - finally: - if stop_after: - containers.stop() - - - def get_docker_in_docker() -> Container: - """ - Grab the parent container named the same as the current machine's hostname. We are assuming that this - is going to be a container. - """ - # There needs to be a test to make sure that the current container (assuming run in dev environment) - # is able to be run from the docker dev environment. That environment is going to be assumed to be - # the same name as the host name of the container. See the docker-compose starting the dev environment - hostname = str(open("/etc/hostname", "rt").read().strip()) - client = docker.from_env() - try: - parent_container = client.containers.get(hostname) - _log.info(f"Inside parent container: {hostname}") - # Setup to use gridappsd as the connection address. This value is used in the - # utils script to establish connection with the gridappsd server - os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" - except docker.errors.NotFound: - _log.debug(f"Docker container is not named this hostname {hostname}") - parent_container = None - return parent_container +# #!/usr/bin/env python3 + +# import contextlib +# import logging +# import os +# import random +# import re +# import shutil +# import tarfile +# import threading +# import time +# import urllib.request +# from copy import deepcopy +# from datetime import datetime, timedelta +# from pathlib import Path +# from pprint import pformat +# from subprocess import PIPE +# from typing import Optional, Union + +# import stomp + +# from gridappsd import GridAPPSD +# from gridappsd.goss import GRIDAPPSD_ENV_ENUM + +# logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) +# logging.getLogger("docker.auth").setLevel(logging.INFO) +# logging.getLogger("docker.utils").setLevel(logging.INFO) + +# _log = logging.getLogger("gridappsd.docker_handler") + +# try: +# from python_on_whales import docker +# HAS_DOCKER = True +# except ImportError: +# _log.warning( +# "Docker api not loaded. pip install docker to install as package.") +# HAS_DOCKER = False + +# if HAS_DOCKER: + +# # The following variable is used for creating a volume for the gridappsd container +# # to utilize. It allows the ability to use multiple containers to run tests +# # along side each other. +# GRIDAPPSD_CONFIG_VOLUME_NAME = f"gridappsd_config_{random.randint(1,100)}" + +# # This named container will be used to hold configuration/folders so that other containers +# # that start can use them. To use add "volume_from": [CONFIGURATION_CONTAINER_NAME] and +# # the mount point within the container will also be within the service container. +# CONFIGURATION_CONTAINER_NAME = "testconfig" + +# def expand_all(user_path): +# return os.path.expandvars(os.path.expanduser(user_path)) + +# __TMP_ROOT__ = "/tmp/assets" +# if Path(__TMP_ROOT__).exists(): +# shutil.rmtree(__TMP_ROOT__, ignore_errors=True) +# os.makedirs(__TMP_ROOT__) + +# # This path needs to be the path to the repo where configuration files are located. +# GRIDAPPSD_CONF_DIR = Path(__file__).parent.parent.joinpath("conf") + +# assert Path(GRIDAPPSD_CONF_DIR).joinpath("entrypoint.sh").exists() +# assert Path(GRIDAPPSD_CONF_DIR).joinpath("run-gridappsd.sh").exists() + +# GRIDAPPSD_DATA_REPO = str(Path(__TMP_ROOT__).joinpath("mysql").resolve()) + +# os.makedirs(GRIDAPPSD_DATA_REPO, exist_ok=True) +# if not Path(GRIDAPPSD_DATA_REPO).is_dir(): +# raise AttributeError( +# f"Invalid GRIDAPPSD_DATA_REPO couldn't make or doesn't exist {GRIDAPPSD_DATA_REPO}" +# ) + +# MYSQL_SCHEMA_INIT_DIR = f'{GRIDAPPSD_DATA_REPO}/docker-entrypoint-initdb.d' + +# def mysql_setup(): +# """ +# Downloads gridappsd_mysql_dump.sql into the MYSQL_SCHEMA_INIT_DIR init directory. +# This will then be mounted into the mysql container. +# """ +# # Downlaod mysql file +# _log.debug("Downloading mysql data file from Bootstrap repository") +# mysql_file = f'{MYSQL_SCHEMA_INIT_DIR}/gridappsd_mysql_dump.sql' +# if os.path.isdir(mysql_file): +# raise RuntimeError( +# f"mysql datafile is directory, delete {mysql_file} using sudo rm -rf {mysql_file}" +# ) +# if not os.path.isdir(MYSQL_SCHEMA_INIT_DIR): +# os.makedirs(MYSQL_SCHEMA_INIT_DIR, 0o0775, exist_ok=True) +# urllib.request.urlretrieve( +# 'https://raw.githubusercontent.com/GRIDAPPSD/Bootstrap/master/gridappsd_mysql_dump.sql', +# filename=mysql_file) + +# # Modify the mysql file to allow connections from gridappsd container +# with open(mysql_file, "r") as sources: +# lines = sources.readlines() +# with open(mysql_file, "w") as sources: +# for line in lines: +# sources.write(re.sub(r'localhost', '%', line)) +# assert Path(mysql_file).exists() + +# # Use the environmental variable if specified otherwise use the develop tag. +# DEFAULT_GRIDAPPSD_TAG = os.environ.get('GRIDAPPSD_TAG_ENV', 'develop') + +# # Network to connect all of the containers up to by default. +# NETWORK = "test_my_network" + +# __TPL_DEPENDENCY_CONFIG__ = { +# "influxdb": { +# "start": True, +# "image": "gridappsd/influxdb:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8086/tcp": 8086 +# }, +# "environment": { +# "INFLUXDB_DB": "proven" +# } +# }, +# "redis": { +# "start": True, +# "image": "redis:3.2.11-alpine", +# "pull": True, +# "ports": { +# "6379/tcp": 6379 +# }, +# "environment": [], +# "entrypoint": "redis-server --appendonly yes", +# }, +# "blazegraph": { +# "start": True, +# "image": "gridappsd/blazegraph:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8080/tcp": 8889 +# }, +# "environment": [] +# }, +# "mysql": { +# "start": +# True, +# "image": +# "mysql/mysql-server:5.7", +# "pull": +# True, +# "ports": { +# "3306/tcp": 3306 +# }, +# "environment": { +# "MYSQL_RANDOM_ROOT_PASSWORD": "yes", +# "MYSQL_PORT": "3306" +# }, +# # "volumes": { +# # "/home/gridappsd/test-assets": {"bind": "/whatthehell", "mode": "rw"} +# # #"test-assets": {"bind": "/whatthehell", "mode": "rw"} +# # #data_dir + "/dumps/": {"bind": "/docker-entrypoint-initdb.d/", "mode": "ro"} +# # }, +# # Our own so we can create +# "volumes_required": [ +# dict(name="mysql_config", +# local_path=MYSQL_SCHEMA_INIT_DIR, +# container_path="/docker-entrypoint-initdb.d") +# ], +# "volumes_from": ["mysql_config"], +# "onsetupfn": +# mysql_setup +# }, +# "proven": { +# "start": True, +# "image": "gridappsd/proven:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": True, +# "ports": { +# "8080/tcp": 18080 +# }, +# "environment": { +# "PROVEN_SERVICES_PORT": "18080", +# "PROVEN_SWAGGER_HOST_PORT": "localhost:18080", +# "PROVEN_USE_IDB": "true", +# "PROVEN_IDB_URL": "http://influxdb:8086", +# "PROVEN_IDB_DB": "proven", +# "PROVEN_IDB_RP": "autogen", +# "PROVEN_IDB_USERNAME": "root", +# "PROVEN_IDB_PASSWORD": "root", +# "PROVEN_T3DIR": "/proven" +# }, +# "links": { +# "influxdb": "influxdb" +# } +# } +# } + +# __TPL_GRIDAPPSD_CONFIG__ = { +# "gridappsd": { +# "start": +# True, +# "image": +# "gridappsd/gridappsd:{{DEFAULT_GRIDAPPSD_TAG}}", +# "pull": +# True, +# "ports": { +# "61613/tcp": 61613, +# "61614/tcp": 61614, +# "61616/tcp": 61616 +# }, +# "environment": { +# "PATH": +# "/gridappsd/bin:/gridappsd/lib:/gridappsd/services/fncsgossbridge/service:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin", +# "DEBUG": 1, +# "START": 1 +# }, +# "links": { +# "mysql": "mysql", +# "influxdb": "influxdb", +# "blazegraph": "blazegraph", +# "proven": "proven", +# "redis": "redis" +# }, +# "volumes_required": [ +# dict(name=GRIDAPPSD_CONFIG_VOLUME_NAME, +# local_path=GRIDAPPSD_CONF_DIR, +# container_path="/startup/conf") +# ], +# "volumes_from": [GRIDAPPSD_CONFIG_VOLUME_NAME], +# "entrypoint": +# "/startup/conf/entrypoint.sh", +# "command": +# "/startup/conf/entrypoint.sh" +# } +# } + +# def __update_template_data__(data, update_dict): +# data_cpy = deepcopy(data) +# for k, v in data_cpy.items(): +# for u, p in update_dict.items(): +# v['image'] = v['image'].replace(u, p) + +# return data_cpy + +# __replace_dict__ = {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG} +# DEFAULT_DOCKER_DEPENDENCY_CONFIG = __update_template_data__( +# __TPL_DEPENDENCY_CONFIG__, __replace_dict__) +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG = __update_template_data__( +# __TPL_GRIDAPPSD_CONFIG__, __replace_dict__) + +# def update_gridappsd_tag(new_gridappsd_tag): +# """ +# Update the default tag used within the dependency and gridappsd containers to be +# what is specified in the new gridappsd_tag variable +# """ +# global DEFAULT_GRIDAPPSD_TAG + +# DEFAULT_GRIDAPPSD_TAG = new_gridappsd_tag +# _log.info(f"Updated gridappsd docker tag {DEFAULT_GRIDAPPSD_TAG} ") +# __replace_dict__.update( +# {"{{DEFAULT_GRIDAPPSD_TAG}}": DEFAULT_GRIDAPPSD_TAG}) +# DEFAULT_DOCKER_DEPENDENCY_CONFIG.update( +# __update_template_data__(__TPL_DEPENDENCY_CONFIG__, +# __replace_dict__)) +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG.update( +# __update_template_data__(__TPL_GRIDAPPSD_CONFIG__, +# __replace_dict__)) + +# class Containers: +# """ +# This class allows the creation/management of containers created by the gridappsd +# docker process. +# """ +# __client__ = docker.from_env() + +# def __init__(self, container_def): +# self._container_def = container_def + +# @property +# def container_def(self): +# return self._container_def + +# @staticmethod +# def remove_container(name): +# try: +# container = Containers.__client__.containers.get(name) +# container.kill() +# except docker.errors.NotFound: +# _log.debug(f"container {name} not found so couldn't remove.") + +# @staticmethod +# def container_list( +# ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): +# """ +# Provides a wrapper around the listing of docker containers. This function was +# brought about when running from within a docker container. + +# Currently the docker container that is run using docker-compose up from the +# gridappsd-dev-environment is specified as gridappsd_dev, however this can change +# and could potentially be extended to multiple named containers. + +# @param: ignore_list: +# optional container names to not stop :: A string or list of strings +# """ + +# if ignore_list is None: +# ignore_list = [] +# elif isinstance(ignore_list, str): +# ignore_list = [ignore_list] +# containers = [] +# for cont in Containers.__client__.containers.list(): +# if cont.name not in ignore_list: +# containers.append(cont) +# return containers + +# @staticmethod +# def reset_all_containers( +# ignore_list: Optional[Union[str, list]] = "gridappsd_dev"): +# """ +# Provides a wrapper around the resetting of all docker containers. This function was +# brought about when running from within a docker container. + +# Currently the docker container that is run using docker-compose up from the +# gridappsd-dev-environment is specified as gridappsd_dev, however this can change +# and could potentially be extended to multiple named containers. + +# @param: ignore_list: +# optional container names to not stop :: A string or list of strings +# """ +# if ignore_list is None: +# ignore_list = [] +# elif isinstance(ignore_list, str): +# ignore_list = [ignore_list] + +# for cont in Containers.__client__.containers.list(): +# if cont.name not in ignore_list: +# cont.kill() + +# @staticmethod +# def check_required_running(self, config): +# my_config = deepcopy(config) + +# for c in Containers.__client__.containers.list(): +# my_config.pop(c.name, None) + +# assert not my_config, f"The required containers were not satisfied missing {list(my_config.keys())}" + +# @staticmethod +# def create_volume_container(name, +# volume_name, +# mount_in_container_at, +# restart_if_exists: bool = False, +# mode="rw"): + +# _log.info( +# f"Creating container {name} and mounting volume {volume_name} " +# f"at {mount_in_container_at} in container {name}") +# client = docker.from_env() + +# should_create = False +# try: +# cont = Containers.__client__.containers.get(name) +# except docker.errors.NotFound: +# # start container +# should_create = True +# else: +# if restart_if_exists: +# should_create = True +# cont.stop() +# else: +# return cont + +# if should_create: +# kwargs = {} +# kwargs['image'] = "alpine" +# # Only name the containers if remove is on +# kwargs['remove'] = True +# kwargs['name'] = name +# kwargs['detach'] = True +# kwargs['volumes'] = { +# volume_name: { +# "bind": mount_in_container_at, +# "mode": mode +# } +# } +# # keep the container running +# kwargs['command'] = "tail -f /dev/null" +# cont = client.containers.run(**kwargs) +# _log.info(f"New container for volume created {cont.name}") +# # print(f"New container is: {container.name}") +# return cont + +# @staticmethod +# def create_get_network(name: str) -> docker.models.networks.Network: +# try: +# network = Containers.__client__.networks.get(name) +# except docker.errors.NotFound: +# network = Containers.__client__.networks.create( +# name, driver="bridge") + +# return network + +# @staticmethod +# def copy_to(src: str, dst: str): +# """ +# Copy a local directory onto a destination + +# .. note:: + +# Make sure the dst (destination) is in the form container:directory + +# @param: src: The local directory to copy from the host +# @param: dst: The container:destination to copy the src to. +# """ +# _log.debug(f"copying folder from: {src} to {dst}") +# src = str(src) +# # python3.6 can't combine PosixPath and str +# assert os.path.exists(src), f"{src} does not exist!" +# client = docker.from_env() +# name, dst = dst.split(':') +# container = client.containers.get(name) +# assert container is not None +# cwd = os.getcwd() +# os.chdir(os.path.dirname(src)) +# srcname = os.path.basename(src) +# tarfilename = src + '.tar' +# tar = tarfile.open(tarfilename, mode='w') +# try: +# tar.add(srcname) +# finally: +# tar.close() + +# data = open(tarfilename, 'rb').read() +# resp = container.put_archive(os.path.dirname(dst), data) +# os.chdir(cwd) +# _log.debug(f"Response from put_archive {resp}") + +# def start(self): +# _log.info("Starting containers") +# # Containers.create_volume_container(CONFIGURATION_CONTAINER_NAME, "testconfig", "/testconfig") +# #pprint(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) +# client = docker.from_env() +# # print(f"Docker client version: {client.version()}") +# for service, value in self._container_def.items(): +# _log.info(f"Pulling {service} image") +# _log.info( +# f"Pulling {service} : {self._container_def[service]['image']}" +# ) +# client.images.pull(self._container_def[service]['image']) +# try: +# container = client.containers.get(service) +# self._container_def[service]['containerid'] = container.id +# except docker.errors.NotFound: +# _log.debug(f"Couldn't find {service} so continuing on.") + +# # Provide a way to dynamically create things that the container will need +# # on the host system. This is important if we want to create a volume before +# # starting the container up. +# if value.get('onsetupfn'): +# value.get('onsetupfn')() + +# if self._container_def[service].get("volumes_required"): +# for vr in self._container_def[service].get( +# "volumes_required"): +# _log.info( +# f"Creating volume for {service}: name={vr['name']}, " +# f"volume_name={vr['name']}, container_path={vr['container_path']}" +# ) +# Containers.create_volume_container( +# name=vr["name"], +# volume_name=vr["name"], +# mount_in_container_at=vr["container_path"], +# restart_if_exists=True) +# time.sleep(20) +# if vr.get("local_path"): +# _log.debug( +# f"contents of local path({vr.get('local_path')}):\n\t{os.listdir(vr.get('local_path'))}" +# ) +# _log.info( +# f"Copy to mounted volume for {service}: " +# f"local_path={vr['local_path']}, container_path={vr['container_path']}" +# ) +# Containers.copy_to( +# vr["local_path"], +# f'{vr["name"]}:{vr["container_path"]}') +# network = Containers.create_get_network(NETWORK) +# for service, value in self._container_def.items(): +# if self._container_def[service]['start']: +# _log.debug( +# f"Starting {service} : {self._container_def[service]['image']}" +# ) +# kwargs = {} +# kwargs['image'] = self._container_def[service]['image'] +# # Only name the containers if remove is on +# kwargs['remove'] = True +# kwargs['name'] = service +# kwargs['detach'] = True +# kwargs['entrypoint'] = value.get('entrypoint') +# if self._container_def[service].get('entrypoint'): +# kwargs['entrypoint'] = value['entrypoint'] +# if self._container_def[service].get('environment'): +# kwargs['environment'] = value['environment'] +# if self._container_def[service].get('ports'): +# kwargs['ports'] = value['ports'] +# if self._container_def[service].get('volumes'): +# kwargs['volumes'] = value['volumes'] +# if self._container_def[service].get('entrypoint'): +# kwargs['entrypoint'] = value['entrypoint'] +# # if self._container_def[service].get('links'): +# # kwargs['links'] = value['links'] +# if self._container_def[service].get('volumes_from'): +# kwargs['volumes_from'] = value['volumes_from'] +# #for k, v in kwargs.items(): +# # print(f"k->{k}, v->{v}") +# _log.debug("Starting container with the following args:") +# _log.debug(f"{pformat(kwargs)}") +# launched_container = None +# try: +# container = client.containers.get(service) +# _log.debug("Found existing container") +# except docker.errors.NotFound: +# container = client.containers.run(**kwargs) +# _log.debug("Started new container") +# network.connect(container.id) +# self._container_def[service]['containerid'] = container.id +# _log.debug( +# f"Current containers are: {[x.name for x in client.containers.list()]}" +# ) + +# def wait_for_log_pattern(self, container, pattern, timeout=60): +# assert self._container_def.get( +# container), f"Container {container} is not in definition." +# client = docker.from_env() +# container = client.containers.get( +# self._container_def.get(container)['containerid']) +# until = datetime.now() + timedelta(seconds=timeout) +# for p in container.logs(stream=True, until=until): +# _log.info(f"HANDLER: {p.decode('utf-8')}") +# if pattern in p.decode('utf-8'): +# print(f"Found pattern {pattern}") +# return +# raise TimeoutError( +# f"Pattern {pattern} was not found in logs of container {container} within {timeout}s" +# ) + +# def wait_for_http_ok(self, url, timeout=30): +# import requests +# results = None +# test_count = 0 + +# while results is None or not results.ok: +# test_count += 1 +# if test_count > timeout: +# raise TimeoutError( +# f"Could not reach {url} within allotted timeout {timeout}s" +# ) +# results = requests.get(url) +# time.sleep(1) + +# print(f"Found url {url} within timeout {timeout}") + +# def stop(self): +# client = docker.from_env() +# for service, value in self._container_def.items(): +# if value.get('containerid'): +# try: +# cnt = client.containers.get(value.get('containerid')) +# cnt.kill() +# time.sleep(2) + +# if "volumes_required" in value: +# # Loop over the volumes that are required for each image and +# # remove the volume. +# for volume_spec in value["volumes_required"]: +# Containers.remove_container( +# volume_spec['name']) +# time.sleep(2) + +# # client.containers.get(value.get('containerid')).kill() # value.get('name')).kill() +# except docker.errors.NotFound as ex: +# _log.error( +# f"Volume {value.get('containerid')} was not found." +# ) +# _log.exception(ex) + +# threads = [] + +# def stream_container_log_to_file(container_name: str, logfile: str): +# client = docker.from_env() +# container = client.containers.list(filters=dict( +# name=container_name))[0] + +# print(container) + +# def log_output(): +# nonlocal container, logfile +# print(f"Starting to write to file {logfile}.") +# with open(logfile, 'wb') as fp: +# print(f"openfile") +# for p in container.logs(stream=True, stderr=PIPE, stdout=PIPE): +# fp.write(p) +# fp.flush() + +# threading.Thread(target=log_output, daemon=True).start() + +# @contextlib.contextmanager +# def run_containers(config, stop_after=True) -> Containers: +# containers = Containers(config) + +# containers.start() +# try: +# yield containers +# finally: +# if stop_after: +# containers.stop() + +# @contextlib.contextmanager +# def run_dependency_containers(stop_after=False) -> Containers: + +# containers = Containers(DEFAULT_DOCKER_DEPENDENCY_CONFIG) + +# containers.start() +# try: +# yield containers +# finally: +# if stop_after: +# containers.stop() + +# @contextlib.contextmanager +# def run_gridappsd_container(stop_after=True, +# rebuild_if_present=False) -> Container: +# """ A contextmanager that uses """ + +# parent_container = get_docker_in_docker() + +# client = docker.from_env() +# # if there is a parent_container then we need to make sure that it is connected +# # to the same network as our systems. If not then we need to modify the network +# # to include the parent container +# if parent_container: +# env = DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd'].get( +# 'environment') +# if env is None: +# env = {} +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_ADDRESS.name] = 'gridappsd' +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_USER.name] = 'test_app_user' +# env[GRIDAPPSD_ENV_ENUM.GRIDAPPSD_PASSWORD.name] = '4Test' +# DEFAULT_GRIDAPPSD_DOCKER_CONFIG['gridappsd']['environment'] = env + +# _log.debug( +# f"Running inside a container environment: {parent_container.name}" +# ) +# network = client.networks.get(NETWORK) +# has_it = False +# for x in network.containers: +# if x.name == parent_container.name: +# has_it = True +# _log.debug( +# f"parent_container {parent_container.name} is connected to the network." +# ) +# break +# if not has_it: +# _log.debug( +# f"Connecting new container to the network: {parent_container.name}" +# ) +# network.connect(parent_container) +# else: +# _log.debug("Not running in a container") + +# containers = Containers(DEFAULT_GRIDAPPSD_DOCKER_CONFIG) + +# gridappsd_container = None +# try: +# gridappsd_container = client.containers.get("gridappsd") +# _log.info(f"{gridappsd_container.name} container found") +# if rebuild_if_present: +# gridappsd_container.kill() +# except docker.errors.NotFound: +# _log.debug("gridappsd container not found!") + +# try: +# if gridappsd_container is None: +# containers.start() + +# # the gridappsd container itself will take a bit to start up. +# time.sleep(30) + +# tries = 30 +# while True: +# tries -= 1 +# if tries <= 0: +# raise RuntimeError( +# "Couldn't connect to gridappsd server in a timely manner!" +# ) +# try: +# g = GridAPPSD() +# if g.connected: +# _log.info("Connected to gridappsd!") +# g.disconnect() +# break + +# except stomp.exception.ConnectFailedException or stomp.exception.NotConnectedException: +# _log.error("Retesting connection") + +# yield gridappsd_container +# finally: +# if stop_after: +# containers.stop() + +# def get_docker_in_docker() -> Container: +# """ +# Grab the parent container named the same as the current machine's hostname. We are assuming that this +# is going to be a container. +# """ +# # There needs to be a test to make sure that the current container (assuming run in dev environment) +# # is able to be run from the docker dev environment. That environment is going to be assumed to be +# # the same name as the host name of the container. See the docker-compose starting the dev environment +# hostname = str(open("/etc/hostname", "rt").read().strip()) +# client = docker.from_env() +# try: +# parent_container = client.containers.get(hostname) +# _log.info(f"Inside parent container: {hostname}") +# # Setup to use gridappsd as the connection address. This value is used in the +# # utils script to establish connection with the gridappsd server +# os.environ["GRIDAPPSD_ADDRESS"] = "gridappsd" +# except docker.errors.NotFound: +# _log.debug( +# f"Docker container is not named this hostname {hostname}") +# parent_container = None +# return parent_container diff --git a/gridappsd-python-lib/info/CHANGELOG.md b/gridappsd-python-lib/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/gridappsd-python-lib/info/VERSION b/gridappsd-python-lib/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/gridappsd-python-lib/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/gridappsd-python-lib/pyproject.toml b/gridappsd-python-lib/pyproject.toml index 432b471..40cafad 100644 --- a/gridappsd-python-lib/pyproject.toml +++ b/gridappsd-python-lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gridappsd-python" -version = "v2.8.230413" +version = "2023.5.2a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", @@ -34,13 +34,16 @@ PyYAML = "^6.0" pytz = "^2022.7" dateutils = "^0.6.7" stomp-py = "6.0.0" +requests = "2.28.2" [tool.poetry.group.dev.dependencies] pytest = "^6.2.2" pytest-html = "^3.1.1" mock = "^4.0.3" -docker = "^4.4.4" yapf = "^0.32.0" +mypy = "^1.3.0" +python-on-whales = "^0.60.1" +gitpython = "^3.1.31" [build-system] requires = ["poetry-core>=1.2.0"] diff --git a/info/CHANGELOG.md b/info/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/info/VERSION b/info/VERSION new file mode 100644 index 0000000..17c91dc --- /dev/null +++ b/info/VERSION @@ -0,0 +1 @@ +2023.5.1 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e6a22c1..f8a2099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] -name = "gridappsd-python" -version = "v2.8.230413" +name = "gridappsd-python-workspace" +version = "2023.5.2a0" description = "A GridAPPS-D Python Adapter" authors = [ "C. Allwardt <3979063+craig8@users.noreply.github.com>", diff --git a/scripts/create_local_version.sh b/scripts/create_local_version.sh new file mode 100755 index 0000000..8e4bcfe --- /dev/null +++ b/scripts/create_local_version.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Usus dunamai to determine a semver compatible version for the current state of the project +# Useefull when building wheels in CI/CD on branches or merge requests, +# without possibly overwriting released versions (of certain tag) +# Used to run in CI/CD, as it will modify both pyproject.toml's and python files (by setting the right string in `__version__=..`) +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# first run directly, to have script stop if dunamai isn't available (for example if not installed, or running in wrong virtual env) +dunamai from any +VERSION=$(dunamai from any) +echo $VERSION + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi +for p in $_projects +do + echo "Creating local version of ${p}" + echo "$VERSION" > "${p}/VERSION" + sed -i$SEP'' "s/^version = .*/version = \"$VERSION\"/" "$p/pyproject.toml" +done +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" package-a/package_a/__init__.py +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" package-b/package_b/__init__.py +sed -i$SEP'' "s/^__version__.*/__version__ = \"$VERSION\"/" service-c/service_c/__init__.py diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh new file mode 100755 index 0000000..3a909f5 --- /dev/null +++ b/scripts/poetry_build.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# This script builds all the poetry packages, creating wheels, dists, and requirements.txt's +# All the wheels will be placed in both the root folder's dist, and in a dist folder within each package +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +poetry version +VERSION=$(poetry version | awk '{print $2}') + +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=$PROJECTS +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + # change path deps in project def + sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml + # include project changelog + cp ../CHANGELOG.md ./ + poetry build + # export deps, with updated path deps + mkdir -p info + poetry export -f requirements.txt --output ./info/requirements.txt --without-hashes --with-credentials + sed -i$SEP'' "s/ @ .*;/==$VERSION;/" "./info/requirements.txt" + ls -altr ./dist/ +done + +# -u for update +if [ "$(uname)" = "Darwin" ]; then export FLAG=" "; else FLAG="-u "; fi +echo "==========" +mkdir -p "${DIR}/../info" +cp $FLAG "${DIR}/../CHANGELOG.md" "${DIR}/../info/" +cp $FLAG "${DIR}/../VERSION" "${DIR}/../info/" +echo "==========" +# copying each wheel to root folder dist +mkdir -p "${DIR}/../dist" +for p in $_projects +do + ls -altr "${DIR}/../${p}/dist/" + cp $FLAG "${DIR}/../${p}/dist/"*".whl" "${DIR}/../dist/" + cp $FLAG "${DIR}/../${p}/dist/"*".tar.gz" "${DIR}/../dist/" +done +echo "==========" +ls -altr "${DIR}/../dist/" +# then copying these to each project +# for p in $_projects +# do +# cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" +# cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" +# ls -altr "${DIR}/../${p}/dist/" +# done diff --git a/scripts/poetry_install.sh b/scripts/poetry_install.sh new file mode 100755 index 0000000..f4a0bc5 --- /dev/null +++ b/scripts/poetry_install.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# This script reflects the latest changes of pyproject.toml +# into both the poetry.lock file and the virtualenv. +# by running `poetry lock --no-update && poetry install --sync` +# It first configures poetry to use the right python for creation of the virtual env +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + poetry env use $(which python3) || poetry env use 3.8 + poetry lock --no-update && poetry install --sync +done diff --git a/scripts/poetry_update.sh b/scripts/poetry_update.sh new file mode 100755 index 0000000..4df08a9 --- /dev/null +++ b/scripts/poetry_update.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# This script reflects the latest changes of pyproject.toml +# into both the poetry.lock file and the virtualenv. +# by running `poetry update && poetry install --sync` +# It first configures poetry to use the right python for creation of the virtual env +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + poetry env use $(which python3) || poetry env use 3.8 + poetry update && poetry install --sync +done diff --git a/scripts/projects.sh b/scripts/projects.sh new file mode 100644 index 0000000..531e65c --- /dev/null +++ b/scripts/projects.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# all python packages, in topological order +PROJECTS='gridappsd-field-bus-lib gridappsd-python-lib' \ No newline at end of file diff --git a/scripts/replace_path_deps.sh b/scripts/replace_path_deps.sh new file mode 100755 index 0000000..0795803 --- /dev/null +++ b/scripts/replace_path_deps.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# This file shows how the sdist & wheel files can be manually modified afterwards +# to replace path dependencies with a version range. +# If the mono repo is at version 1.2.3, it will set the dependencies to (~=1.2,>=1.2.3), effectively equal to ~1.2.3. +set -x +set -u +set -e + +VERSION=$(poetry version | awk '{print $2}') +VERSION_MINOR=$(echo $VERSION | sed -E "s/^([0-9]*\.[0-9]*).*/\1/") +curdir=$(pwd) +if [ "$(uname)" = "Darwin" ]; then export SEP=" "; else SEP=""; fi + +# ===== Updating the TAR.GZ file ===== +cd "$curdir" +TARFILES=$(ls dist/*.tar.gz) +for TARFILE in $TARFILES +do + rm -rf /tmp/version_update + mkdir -p /tmp/version_update + tar -C /tmp/version_update -xf $curdir/$TARFILE + cd /tmp/version_update + # Replace the path dependencies (which are prefixed with '@') + # with compatible version to the current monorepo, but at least at the current one. + # In semver notation: ~1.2.3, which equals >=1.2.3, <2.0.0 + # Note that allowed matches are defined at: + # https://peps.python.org/pep-0440/#compatible-release + # We therefore specify that we require >=1.2.3 AND <2.0 + # Thus at least at the same fix version, but only compatible versions. + # Therefore we use ~=1.2, which equals >=1.2,<2.0, together with >=1.2.3 + FOLDER=$(ls) + sed -i$SEP'' "s|^Requires-Dist: \(.*\) @ \.\./.*|Requires-Dist: \1 (~=$VERSION_MINOR,>=$VERSION)|" "$FOLDER/PKG-INFO" + sed -i$SEP'' "s| @ \.\.[a-zA-Z\-_/]*|~=$VERSION_MINOR,>=$VERSION|" "$FOLDER/setup.py" + sed -i$SEP'' "s|{.*path.*\.\..*|\"~$VERSION\"|" "$FOLDER/pyproject.toml" + tar -czvf new.tar.gz "$FOLDER" + mv new.tar.gz $curdir/$TARFILE +done + +# ===== Updating the WHEEL file ===== +# Handle the tar.gz +cd "$curdir" +WHEELFILES=$(ls dist/*.whl) +for WHEELFILE in $WHEELFILES +do + rm -rf /tmp/version_update + mkdir -p /tmp/version_update + unzip -d /tmp/version_update $curdir/$WHEELFILE + cd /tmp/version_update + # Replace the path dependencies (which are prefixed with '@') + # with compatible version to the current monorepo, but at least at the current one. + # In semver notation: ~1.2.3, which equals >=1.2.3, <2.0.0 + # Note that allowed matches are defined at: + # https://peps.python.org/pep-0440/#compatible-release + # We therefore specify that we require >=1.2.3 AND <2.0 + # Thus at least at the same fix version, but only compatible versions. + # Therefore we use ~=1.2, which equals >=1.2,<2.0, together with >=1.2.3 + FOLDER=$(ls -d *.dist-info) + sed -i$SEP'' "s|^Requires-Dist: \(.*\) @ \.\./.*|Requires-Dist: \1 (~=$VERSION_MINOR,>=$VERSION)|" "$FOLDER/METADATA" + zip -r new.whl ./* + mv new.whl "$curdir/$WHEELFILE" + cd "$curdir" + rm -rf /tmp/version_update +done diff --git a/scripts/run_on_each.sh b/scripts/run_on_each.sh new file mode 100755 index 0000000..2722660 --- /dev/null +++ b/scripts/run_on_each.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# runs the passed command in each poetry project folder +set -x +set -u +set -e +DIR="$( cd "$( dirname "$0" )" && pwd )" +cd "${DIR}/.." || exit + +# all python packages, in topological order +. ${DIR}/projects.sh +_projects=". ${PROJECTS}" +echo "Running on following projects: ${_projects}" +for p in $_projects +do + cd "${DIR}/../${p}" || exit + echo "==running in ${p}==" + "$@" +done From 4d0f7df2d2cb250e7f78dd07ea4bf2b6f6d4b355 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:31:32 -0700 Subject: [PATCH 12/33] Add poetry.toml file for in-project virtual envs --- gridappsd-field-bus-lib/poetry.toml | 2 ++ gridappsd-python-lib/poetry.toml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 gridappsd-field-bus-lib/poetry.toml create mode 100644 gridappsd-python-lib/poetry.toml diff --git a/gridappsd-field-bus-lib/poetry.toml b/gridappsd-field-bus-lib/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/gridappsd-field-bus-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/gridappsd-python-lib/poetry.toml b/gridappsd-python-lib/poetry.toml new file mode 100644 index 0000000..ab1033b --- /dev/null +++ b/gridappsd-python-lib/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true From 2805500652f4b00ee1411a0d1c3b86bbc9677613 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:35:08 -0700 Subject: [PATCH 13/33] Test for monorepo granch --- .github/workflows/deploy-dev-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 20679a8..59863a6 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -39,8 +39,8 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - ref: develop - token: ${{ secrets.git-token }} + # ref: develop + token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python ${{ env.PYTHON_VERSION }} id: setup-python From 86e18dbe27b497eb59b05164290d655dac25b41a Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:38:25 -0700 Subject: [PATCH 14/33] Add copying to all dirs back --- scripts/poetry_build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh index 3a909f5..4e68551 100755 --- a/scripts/poetry_build.sh +++ b/scripts/poetry_build.sh @@ -49,9 +49,9 @@ done echo "==========" ls -altr "${DIR}/../dist/" # then copying these to each project -# for p in $_projects -# do -# cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" -# cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" -# ls -altr "${DIR}/../${p}/dist/" -# done +for p in $_projects +do + cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" + cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" + ls -altr "${DIR}/../${p}/dist/" +done From 3a4cb4006dcf3be326fd423484f9c3d3c119ac0f Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:42:43 -0700 Subject: [PATCH 15/33] Use path reference only! --- gridappsd-field-bus-lib/info/requirements.txt | 22 +++++++++++++++++++ gridappsd-field-bus-lib/pyproject.toml | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 gridappsd-field-bus-lib/info/requirements.txt diff --git a/gridappsd-field-bus-lib/info/requirements.txt b/gridappsd-field-bus-lib/info/requirements.txt new file mode 100644 index 0000000..2e37a34 --- /dev/null +++ b/gridappsd-field-bus-lib/info/requirements.txt @@ -0,0 +1,22 @@ +certifi==2023.5.7 ; python_full_version >= "3.7.9" and python_version < "4" +charset-normalizer==3.1.0 ; python_full_version >= "3.7.9" and python_version < "4" +cim-graph==0.1.20230508194360a0 ; python_full_version >= "3.7.9" and python_version < "4.0" +dateutils==0.6.12 ; python_full_version >= "3.7.9" and python_version < "4.0" +docopt==0.6.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +gridappsd-python==2023.5.2a0; python_full_version >= "3.7.9" and python_version < "4.0" +idna==3.4 ; python_full_version >= "3.7.9" and python_version < "4" +importlib-metadata==4.13.0 ; python_full_version >= "3.7.9" and python_version < "3.8" +isodate==0.6.1 ; python_full_version >= "3.7.9" and python_version < "4.0" +pyparsing==3.0.9 ; python_full_version >= "3.7.9" and python_version < "4.0" +python-dateutil==2.8.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +pytz==2022.7.1 ; python_full_version >= "3.7.9" and python_version < "4.0" +pyyaml==6.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +rdflib==6.3.2 ; python_full_version >= "3.7.9" and python_version < "4.0" +requests==2.28.2 ; python_full_version >= "3.7.9" and python_version < "4" +six==1.16.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +sparqlwrapper==2.0.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +stomp-py==6.0.0 ; python_full_version >= "3.7.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_full_version >= "3.7.9" and python_version < "3.8" +urllib3==1.26.15 ; python_full_version >= "3.7.9" and python_version < "4" +xsdata==22.12 ; python_full_version >= "3.7.9" and python_version < "4.0" +zipp==3.15.0 ; python_full_version >= "3.7.9" and python_version < "3.8" diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 0c9ecd6..5434bf6 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -24,7 +24,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.7.9,<4.0" -gridappsd-python = "^2023.5.2a0" +gridappsd-python = {path="../gridappsd-python-lib", develop=true} cim-graph = "^0.1.20230413185916a0" [tool.poetry.group.dev.dependencies] From d2cfbd063e72c2d23afe43655dce5592f16fe8dd Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:43:43 -0700 Subject: [PATCH 16/33] Don't copy to each project --- scripts/poetry_build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh index 4e68551..3a909f5 100755 --- a/scripts/poetry_build.sh +++ b/scripts/poetry_build.sh @@ -49,9 +49,9 @@ done echo "==========" ls -altr "${DIR}/../dist/" # then copying these to each project -for p in $_projects -do - cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" - cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" - ls -altr "${DIR}/../${p}/dist/" -done +# for p in $_projects +# do +# cp $FLAG "${DIR}/../dist/"*".whl" "${DIR}/../${p}/dist/" +# cp $FLAG "${DIR}/../info/"*"" "${DIR}/../${p}/info/" +# ls -altr "${DIR}/../${p}/dist/" +# done From 6fb30dae8fe683f0f94d82ba3894b3207b013364 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:46:41 -0700 Subject: [PATCH 17/33] Don't change path to release version, leave as path --- scripts/poetry_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh index 3a909f5..467279b 100755 --- a/scripts/poetry_build.sh +++ b/scripts/poetry_build.sh @@ -20,7 +20,8 @@ for p in $_projects do cd "${DIR}/../${p}" || exit # change path deps in project def - sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml + echo "Leave the path to local version" + #sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml # include project changelog cp ../CHANGELOG.md ./ poetry build From 2a8940d817c920d874ca658b8ed2579cf4353f20 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:51:16 -0700 Subject: [PATCH 18/33] Move -x -u -e to top of block --- .github/workflows/deploy-dev-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 59863a6..a34cde4 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -181,14 +181,14 @@ jobs: - name: Create build artifacts run: | - # set the right version in pyproject.toml before build and publish - ./scripts/poetry_build.sh - ./scripts/replace_path_deps.sh - set -x set -u set -e + # set the right version in pyproject.toml before build and publish + ./scripts/poetry_build.sh + ./scripts/replace_path_deps.sh + # all python packages, in topological order . scripts/projects.sh _projects=". ${PROJECTS}" From dad58ef2893f0cbe819f1511d1f2d315a3a9c254 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 11:59:16 -0700 Subject: [PATCH 19/33] Update from testing locally --- .github/workflows/deploy-dev-release.yml | 24 ++++++++++++------------ scripts/poetry_build.sh | 3 +-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index a34cde4..c02a75d 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -187,18 +187,18 @@ jobs: # set the right version in pyproject.toml before build and publish ./scripts/poetry_build.sh - ./scripts/replace_path_deps.sh - - # all python packages, in topological order - . scripts/projects.sh - _projects=". ${PROJECTS}" - echo "Running on following projects: ${_projects}" - for p in $_projects - do - cd "${p}" || exit - echo "==running in ${p}==" - ../scripts/replace_path_deps.sh - done + #./scripts/replace_path_deps.sh + + # # all python packages, in topological order + # . scripts/projects.sh + # _projects=". ${PROJECTS}" + # echo "Running on following projects: ${_projects}" + # for p in $_projects + # do + # cd "${p}" || exit + # echo "==running in ${p}==" + # ../scripts/replace_path_deps.sh + # done # poetry version ${NEW_TAG#?} # poetry build -vvv diff --git a/scripts/poetry_build.sh b/scripts/poetry_build.sh index 467279b..3a909f5 100755 --- a/scripts/poetry_build.sh +++ b/scripts/poetry_build.sh @@ -20,8 +20,7 @@ for p in $_projects do cd "${DIR}/../${p}" || exit # change path deps in project def - echo "Leave the path to local version" - #sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml + sed -i$SEP'' "s|{.*path.*|\"^$VERSION\"|" pyproject.toml # include project changelog cp ../CHANGELOG.md ./ poetry build From 912ada77d34a8cfd060e085a23e27381e8832fc6 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:02:51 -0700 Subject: [PATCH 20/33] Only build docker if we are on the right repo. --- .github/workflows/main.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34b781a..0438b96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,7 @@ on: [push, pull_request] jobs: push: + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' runs-on: ubuntu-latest name: Build and push the docker container steps: @@ -19,7 +20,7 @@ jobs: - name: Log in to docker run: | if [ -n "${{ secrets.DOCKER_USERNAME }}" -a -n "${{ secrets.DOCKER_TOKEN }}" ]; then - + echo " " echo "Connecting to docker" echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin @@ -29,7 +30,7 @@ jobs: exit 1 fi fi - + - name: Build the image env: DOCKER_IMAGE_NAME: ${{ secrets.DOCKER_IMAGE_NAME }} @@ -67,11 +68,11 @@ jobs: ORG="${ORG:+${ORG}/}" IMAGE="${ORG}${{ env.DOCKER_IMAGE_NAME }}" if [ -n "${{ secrets.DOCKER_USERNAME }}" -a -n "${{ secrets.DOCKER_TOKEN }}" ]; then - + if [ -n "$TAG" -a -n "$ORG" ]; then # Get the built container name CONTAINER=`docker images --format "{{.Repository}}:{{.Tag}}" ${IMAGE}` - + echo "docker push ${CONTAINER}" docker push "${CONTAINER}" status=$? @@ -79,7 +80,7 @@ jobs: echo "Error: status $status" exit 1 fi - + echo "docker tag ${CONTAINER} ${IMAGE}:$TAG" docker tag ${CONTAINER} ${IMAGE}:$TAG status=$? @@ -87,7 +88,7 @@ jobs: echo "Error: status $status" exit 1 fi - + echo "docker push ${IMAGE}:$TAG" docker push ${IMAGE}:$TAG status=$? @@ -96,5 +97,5 @@ jobs: exit 1 fi fi - + fi From 3389ec01af827f830a175809853acc5dc1e090d6 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:05:23 -0700 Subject: [PATCH 21/33] update to release new artfifact to branch --- .github/workflows/deploy-dev-release.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index c02a75d..515646a 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -23,6 +23,7 @@ env: jobs: deploy-dev-release: + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' runs-on: ubuntu-22.04 permissions: contents: write # To push a branch @@ -203,16 +204,16 @@ jobs: # poetry version ${NEW_TAG#?} # poetry build -vvv - # - uses: ncipollo/release-action@v1 - # with: - # artifacts: "dist/*.gz,dist/*.whl" - # artifactErrorsFailBuild: true - # generateReleaseNotes: true - # commit: ${{ github.ref }} - # # check bump-rule and set accordingly - # prerelease: true - # tag: ${{ env.NEW_TAG }} - # token: ${{ secrets.git-token }} + - uses: ncipollo/release-action@v1 + with: + artifacts: "dist/*.gz,dist/*.whl" + artifactErrorsFailBuild: true + generateReleaseNotes: true + commit: ${{ github.ref }} + # check bump-rule and set accordingly + prerelease: true + tag: ${{ env.NEW_TAG }} + token: ${{ secrets.GITHUB_TOKEN }} # - name: Publish to pypi # id: publish-to-pypi From 7f4f181bcf203688d548217ddf0ead2c48304435 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:08:35 -0700 Subject: [PATCH 22/33] Name the artifacts release --- .github/workflows/deploy-dev-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 515646a..64ef87b 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -204,7 +204,8 @@ jobs: # poetry version ${NEW_TAG#?} # poetry build -vvv - - uses: ncipollo/release-action@v1 + - name: Push artifacts to github + uses: ncipollo/release-action@v1 with: artifacts: "dist/*.gz,dist/*.whl" artifactErrorsFailBuild: true From 857a73d0d6ac9c4e4f567b76e15851d1c2675c19 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:10:10 -0700 Subject: [PATCH 23/33] Remove the if condition for development --- .github/workflows/deploy-dev-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 64ef87b..25c69bd 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -23,7 +23,6 @@ env: jobs: deploy-dev-release: - if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' runs-on: ubuntu-22.04 permissions: contents: write # To push a branch From 186fa4a6bf74e38b7d65f1a298d90741eead4f94 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:18:05 -0700 Subject: [PATCH 24/33] bump version needs to run in order to do prelease --- .github/workflows/deploy-dev-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 25c69bd..7864821 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -122,6 +122,7 @@ jobs: # current_tag is now the version we want to set our poetry version so # that we can bump the version ./scripts/run_on_each.sh poetry version ${current_tag} + ./scripts/run_on_each.sh poetry version prerelease # poetry version ${current_tag} # poetry version prerelease --no-interaction From c33f4763462f843edda897be55078e46183cb6aa Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 17 May 2023 12:18:59 -0700 Subject: [PATCH 25/33] Add pypi push back for gridappsd and cim-tools --- .github/workflows/deploy-dev-release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 7864821..1769243 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -216,13 +216,13 @@ jobs: tag: ${{ env.NEW_TAG }} token: ${{ secrets.GITHUB_TOKEN }} - # - name: Publish to pypi - # id: publish-to-pypi - # if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' - # run: | - # echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - # ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.pypi-token }} - # ./scripts/run_on_each.sh poetry publish + - name: Publish to pypi + id: publish-to-pypi + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' + run: | + echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV + ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.pypi-token }} + ./scripts/run_on_each.sh poetry publish # - name: Publish to test-pypi # id: publish-to-test-pypi From 5df92c6c7f4ebe6d263690b1fac4b662cacb7462 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 18 May 2023 14:35:22 -0700 Subject: [PATCH 26/33] Corrected PYPI_TOKEN --- .github/workflows/deploy-dev-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 1769243..074ca23 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -1,4 +1,4 @@ -name: Deploy Release Artifacts +name: Deploy Dev Release Artifacts on: push: @@ -221,7 +221,7 @@ jobs: if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' run: | echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.pypi-token }} + ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} ./scripts/run_on_each.sh poetry publish # - name: Publish to test-pypi From 265155366e8eeab0eea3bf9ec8ce3d9fc564220d Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Fri, 19 May 2023 08:50:24 -0700 Subject: [PATCH 27/33] Updated publish to pypi for dev releases --- .github/workflows/deploy-dev-release.yml | 67 +++++------------------- 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/.github/workflows/deploy-dev-release.yml b/.github/workflows/deploy-dev-release.yml index 074ca23..9534bd7 100644 --- a/.github/workflows/deploy-dev-release.yml +++ b/.github/workflows/deploy-dev-release.yml @@ -142,43 +142,6 @@ jobs: # Finally because we want to be able to use the variable in later # steps we set a NEW_TAG environmental variable echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV - # we don't want to update pyproject.toml yet. don't want this change to create merge conflict. - # we don't really persist right version in pyproject.toml to figure out the next version. we use git tags. - #git restore pyproject.toml - - # #-------------------------------------------------------------- - # # Create a new releases/new_tag - # #-------------------------------------------------------------- - # - name: Create a new releases branch - # run: | - # git checkout -b releases/${NEW_TAG} - # git push --set-upstream origin releases/${NEW_TAG} - - # #-------------------------------------------------------------- - # # merge changes back to main - # #-------------------------------------------------------------- - # - name: Merge changes back to main - # run: | - # git checkout main - # git merge ${{ inputs.merge-strategy }} releases/${NEW_TAG} - # git push - - # - name: Run tests on main branch - # id: run-tests-on-main - # run: | - # poetry run pytest --timeout=${{ inputs.run-tests-wait }} tests - # continue-on-error: true - - # - name: Do something with a failing build - # if: steps.run-tests-on-main.outcome != 'success' - # run: | - # echo "tests on main did not succeed. Outcome is ${{ steps.run-tests-on-main.outcome }}" - # git reset --hard HEAD~1 - # git push origin HEAD --force - # git branch -d releases/${NEW_TAG} - # git push origin --delete releases/${NEW_TAG} - # echo "reverted changes to main and removed release branch" - # exit 1 - name: Create build artifacts run: | @@ -188,21 +151,6 @@ jobs: # set the right version in pyproject.toml before build and publish ./scripts/poetry_build.sh - #./scripts/replace_path_deps.sh - - # # all python packages, in topological order - # . scripts/projects.sh - # _projects=". ${PROJECTS}" - # echo "Running on following projects: ${_projects}" - # for p in $_projects - # do - # cd "${p}" || exit - # echo "==running in ${p}==" - # ../scripts/replace_path_deps.sh - # done - - # poetry version ${NEW_TAG#?} - # poetry build -vvv - name: Push artifacts to github uses: ncipollo/release-action@v1 @@ -220,9 +168,20 @@ jobs: id: publish-to-pypi if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' run: | + set -x + set -u + set -e + + # This is needed, because the poetry publish will fail at the top level of the project + # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV - ./scripts/run_on_each.sh poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - ./scripts/run_on_each.sh poetry publish + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish + + cd ../gridappsd-field-bus-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish # - name: Publish to test-pypi # id: publish-to-test-pypi From 384d47d56a57652bc9e22102a5a56290121eadb8 Mon Sep 17 00:00:00 2001 From: afisher1 Date: Fri, 19 May 2023 10:55:43 -0700 Subject: [PATCH 28/33] fix: #116 --- .../field_interface/agents/agents.py | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py index 0ae2f7c..ff48e57 100644 --- a/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py +++ b/gridappsd-field-bus-lib/gridappsd/field_interface/agents/agents.py @@ -229,9 +229,19 @@ def __init__(self, self).__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, feeder_dict, simulation_id) - + self.feeder_area = None + self.downstream_message_bus_def = downstream_message_bus_def if self.agent_area_dict is not None: - feeder = cim.Feeder(mRID=downstream_message_bus_def.id) + feeder = cim.Feeder(mRID=self.downstream_message_bus_def.id) + self.feeder_area = DistributedModel(connection=self.connection, + feeder=feeder, + topology=self.agent_area_dict) + + + def connect(self): + super().connect() + if self.feeder_area is None: + feeder = cim.Feeder(mRID=self.downstream_message_bus_def.id) self.feeder_area = DistributedModel(connection=self.connection, feeder=feeder, topology=self.agent_area_dict) @@ -245,12 +255,20 @@ def __init__(self, agent_config: Dict, switch_area_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, switch_area_dict, simulation_id) - + self.switch_area = None + self.downstream_message_bus_def = downstream_message_bus_def if self.agent_area_dict is not None: - self.switch_area = SwitchArea(downstream_message_bus_def.id, + self.switch_area = SwitchArea(self.downstream_message_bus_def.id, + self.connection) + self.switch_area.initialize_switch_area(self.agent_area_dict) + + + def connect(self): + super().connect() + if self.switch_area is None: + self.switch_area = SwitchArea(self.downstream_message_bus_def.id, self.connection) self.switch_area.initialize_switch_area(self.agent_area_dict) @@ -263,12 +281,20 @@ def __init__(self, agent_config: Dict, secondary_area_dict=None, simulation_id=None): - super().__init__(upstream_message_bus_def, downstream_message_bus_def, agent_config, secondary_area_dict, simulation_id) - + self.secondary_area = None + self.downstream_message_bus_def = downstream_message_bus_def if self.agent_area_dict is not None: - self.secondary_area = SecondaryArea(downstream_message_bus_def.id, + self.secondary_area = SecondaryArea(self.downstream_message_bus_def.id, + self.connection) + self.secondary_area.initialize_secondary_area(self.agent_area_dict) + + + def connect(self): + super().connect() + if self.secondary_area is None: + self.secondary_area = SecondaryArea(self.downstream_message_bus_def.id, self.connection) self.secondary_area.initialize_secondary_area(self.agent_area_dict) From 96bd39ff62e540a3c89715d934dcf267153fb93d Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 24 May 2023 15:11:29 -0700 Subject: [PATCH 29/33] Update pyproject.toml Update cim-graph --- gridappsd-field-bus-lib/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gridappsd-field-bus-lib/pyproject.toml b/gridappsd-field-bus-lib/pyproject.toml index 5434bf6..6da09ad 100644 --- a/gridappsd-field-bus-lib/pyproject.toml +++ b/gridappsd-field-bus-lib/pyproject.toml @@ -25,7 +25,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.7.9,<4.0" gridappsd-python = {path="../gridappsd-python-lib", develop=true} -cim-graph = "^0.1.20230413185916a0" +cim-graph = "^2023.5.1a3" [tool.poetry.group.dev.dependencies] pytest = "^6.2.2" From e6df45f9c1eb4d97f0bfc8f4a6a1c18d8a139a3f Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 6 Jun 2023 16:48:34 +0000 Subject: [PATCH 30/33] Add release package for releasing real releases --- .github/workflows/release.yml | 148 ++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d1618a0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,148 @@ +name: Deploy Release Artifacts + +on: + workflow_dispatch: + inputs: + previous-version: + description: "Previous version number to use for release notes generation." + required: true + type: bool + release-version: + description: "Version number to use for this release, do not start with v." + required: true + type: bool + publish-to: + description: "Publish to pypi or pypi-test" + required: true + type: string + default: "pypi" + options: + - "pypi" + - "pypi-test" + +defaults: + run: + shell: bash + +env: + LANG: en_US.utf-8 + LC_ALL: en_US.utf-8 + PYTHON_VERSION: "3.10" + +jobs: + deploy-release: + runs-on: ubuntu-22.04 + permissions: + contents: write # To push a branch + pull-requests: write # To create a PR from that branch + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + # ref: develop + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1.3.3 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: | + ./scripts/poetry_install.sh + + #---------------------------------------------- + # Update to new release version + #---------------------------------------------- + - name: Update Version + run: | + ./scripts/run_on_each.sh poetry version ${{ inputs.release-version }} + + NEW_TAG=v$(poetry version --short) + + # Finally because we want to be able to use the variable in later + # steps we set a NEW_TAG environmental variable + echo "NEW_TAG=$(echo ${NEW_TAG})" >> $GITHUB_ENV + + - name: Create build artifacts + run: | + set -x + set -u + set -e + + # set the right version in pyproject.toml before build and publish + ./scripts/poetry_build.sh + + - name: Push artifacts to github + # if: inputs.publish-to == 'pypi' + uses: ncipollo/release-action@v1 + with: + artifacts: "dist/*.gz,dist/*.whl" + artifactErrorsFailBuild: true + generateReleaseNotes: true + commit: ${{ github.ref }} + makeLatest: true + tag: ${{ env.NEW_TAG }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to pypi + id: publish-to-pypi + if: github.repository_owner == 'GRIDAPPSD' || github.repository_owner == 'PNNL-CIM-Tools' + run: | + set -x + set -u + set -e + + # This is needed, because the poetry publish will fail at the top level of the project + # so ./scripts/run_on_each.sh fails for that. + echo "POETRY_PUBLISH_OPTIONS=''" >> $GITHUB_ENV + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish + + cd ../gridappsd-field-bus-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} + poetry publish + + - name: Publish to pypi test + id: publish-to-pypi-test + if: inputs.publish-to == 'pypi-test' + run: | + set -x + set -u + set -e + + ./scripts/run_on_each.sh poetry config repositories.testpypi https://test.pypi.org/legacy/ + + # This is needed, because the poetry publish will fail at the top level of the project + # so ./scripts/run_on_each.sh fails for that. + echo "POETRY_PUBLISH_OPTIONS='--repository testpypi'" >> $GITHUB_ENV + cd gridappsd-python-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry publish + + cd ../gridappsd-field-bus-lib + poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry publish \ No newline at end of file From 740e9cd1935ee927364d4b18d40d24e405553ef0 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 6 Jun 2023 16:53:48 +0000 Subject: [PATCH 31/33] fix choice input for action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1618a0..1eba0aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ on: publish-to: description: "Publish to pypi or pypi-test" required: true - type: string + type: choice default: "pypi" options: - "pypi" From 3a6116422bef8e961567a061eaec4250fc91e53d Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 6 Jun 2023 17:03:06 +0000 Subject: [PATCH 32/33] Fix pypi-token name --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1eba0aa..c721c72 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -140,9 +140,9 @@ jobs: # so ./scripts/run_on_each.sh fails for that. echo "POETRY_PUBLISH_OPTIONS='--repository testpypi'" >> $GITHUB_ENV cd gridappsd-python-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish cd ../gridappsd-field-bus-lib - poetry config pypi-token.pypi ${{ secrets.PYPI_TEST_TOKEN }} + poetry config pypi-token.testpypi ${{ secrets.PYPI_TEST_TOKEN }} poetry publish \ No newline at end of file From 37b84649130289bee319b100bb260c7a66808fec Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 6 Jun 2023 17:09:38 +0000 Subject: [PATCH 33/33] Remove if condition on publish artifacts to github. --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c721c72..dcdd084 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,7 +96,6 @@ jobs: ./scripts/poetry_build.sh - name: Push artifacts to github - # if: inputs.publish-to == 'pypi' uses: ncipollo/release-action@v1 with: artifacts: "dist/*.gz,dist/*.whl"