Skip to content

Commit

Permalink
Download custom appliance symbols from GitHub
Browse files Browse the repository at this point in the history 
Fix symbol cache issue. Ref GNS3/gns3-gui#2671
Fix temporary directory for symbols was not deleted
Fix temporary appliance file was not deleted
  • Loading branch information
@grossmj
grossmj committed Mar 11, 2019
1 parent 889d29e commit bae3fb8
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 37 deletions.
4 changes: 4 additions & 0 deletions gns3server/controller/appliance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ def id(self):
def status(self):
return self._data["status"]

@property
def symbol(self):
return self._data.get("symbol")

def __json__(self):
"""
Appliance data (a hash)
Expand Down
48 changes: 45 additions & 3 deletions gns3server/controller/appliance_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,51 @@ def load_appliances(self):
with open(path, 'r', encoding='utf-8') as f:
appliance = Appliance(appliance_id, json.load(f), builtin=builtin)
appliance.__json__() # Check if loaded without error
if appliance.status != 'broken':
self._appliances[appliance.id] = appliance
if appliance.status != 'broken':
self._appliances[appliance.id] = appliance
except (ValueError, OSError, KeyError) as e:
log.warning("Cannot load appliance file '%s': %s", path, str(e))
continue

async def download_custom_symbols(self):
"""
Download custom appliance symbols from our GitHub registry repository.
"""

from . import Controller
symbol_dir = Controller.instance().symbols.symbols_path()
self.load_appliances()
for appliance in self._appliances.values():
symbol = appliance.symbol
if symbol and not symbol.startswith(":/symbols/"):
destination_path = os.path.join(symbol_dir, symbol)
if not os.path.exists(destination_path):
await self._download_symbol(symbol, destination_path)

# refresh the symbol cache
Controller.instance().symbols.list()

async def _download_symbol(self, symbol, destination_path):
"""
Download a custom appliance symbol from our GitHub registry repository.
"""

symbol_url = "https://raw.githubusercontent.com/GNS3/gns3-registry/master/symbols/{}".format(symbol)
async with aiohttp.ClientSession() as session:
async with session.get(symbol_url) as response:
if response.status != 200:
log.warning("Could not retrieve appliance symbol {} from GitHub due to HTTP error code {}".format(symbol, response.status))
else:
try:
symbol_data = await response.read()
log.info("Saving {} symbol to {}".format(symbol, destination_path))
with open(destination_path, 'wb') as f:
f.write(symbol_data)
except asyncio.TimeoutError:
log.warning("Timeout while downloading '{}'".format(symbol_url))
except OSError as e:
log.warning("Could not write appliance symbol '{}': {}".format(destination_path, e))

@locking
async def download_appliances(self):
"""
Expand All @@ -114,7 +153,7 @@ async def download_appliances(self):
log.info("Appliances are already up-to-date (ETag {})".format(self._appliances_etag))
return
elif response.status != 200:
raise aiohttp.web.HTTPConflict(text="Could not retrieve appliances on GitHub due to HTTP error code {}".format(response.status))
raise aiohttp.web.HTTPConflict(text="Could not retrieve appliances from GitHub due to HTTP error code {}".format(response.status))
etag = response.headers.get("ETag")
if etag:
self._appliances_etag = etag
Expand Down Expand Up @@ -144,3 +183,6 @@ async def download_appliances(self):
raise aiohttp.web.HTTPConflict(text="Could not write appliance file '{}': {}".format(path, e))
except ValueError as e:
raise aiohttp.web.HTTPConflict(text="Could not read appliances information from GitHub: {}".format(e))

# download the custom symbols
await self.download_custom_symbols()
24 changes: 12 additions & 12 deletions gns3server/controller/symbol_themes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"vmware_guest": ":/symbols/classic/vmware_guest.svg",
"docker_guest": ":/symbols/classic/docker_guest.svg"}

INFINITY_SQUARE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/blue/cloud.svg",
AFFINITY_SQUARE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/blue/cloud.svg",
"ethernet_switch": ":/symbols/affinity/square/blue/switch.svg",
"ethernet_hub": ":/symbols/affinity/square/blue/hub.svg",
"frame_relay_switch.svg": ":/symbols/affinity/square/blue/isdn.svg",
Expand All @@ -46,7 +46,7 @@
"vmware_guest": ":/symbols/affinity/square/blue/vmware.svg",
"docker_guest": ":/symbols/affinity/square/blue/docker.svg"}

INFINITY_SQUARE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/red/cloud.svg",
AFFINITY_SQUARE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/red/cloud.svg",
"ethernet_switch": ":/symbols/affinity/square/red/switch.svg",
"ethernet_hub": ":/symbols/affinity/square/red/hub.svg",
"frame_relay_switch": ":/symbols/affinity/square/red/isdn.svg",
Expand All @@ -61,7 +61,7 @@
"vmware_guest": ":/symbols/affinity/square/red/vmware.svg",
"docker_guest": ":/symbols/affinity/square/red/docker.svg"}

INFINITY_SQUARE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/gray/cloud.svg",
AFFINITY_SQUARE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/gray/cloud.svg",
"ethernet_switch": ":/symbols/affinity/square/gray/switch.svg",
"ethernet_hub": ":/symbols/affinity/square/gray/hub.svg",
"frame_relay_switch": ":/symbols/affinity/square/gray/isdn.svg",
Expand All @@ -76,7 +76,7 @@
"vmware_guest": ":/symbols/affinity/square/gray/vmware.svg",
"docker_guest": ":/symbols/affinity/square/gray/docker.svg"}

INFINITY_CIRCLE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/blue/cloud.svg",
AFFINITY_CIRCLE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/blue/cloud.svg",
"ethernet_switch": ":/symbols/affinity/circle/blue/switch.svg",
"ethernet_hub": ":/symbols/affinity/circle/blue/hub.svg",
"frame_relay_switch": ":/symbols/affinity/circle/blue/isdn.svg",
Expand All @@ -91,7 +91,7 @@
"vmware_guest": ":/symbols/affinity/circle/blue/vmware.svg",
"docker_guest": ":/symbols/affinity/circle/blue/docker.svg"}

INFINITY_CIRCLE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/red/cloud.svg",
AFFINITY_CIRCLE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/red/cloud.svg",
"ethernet_switch": ":/symbols/affinity/circle/red/switch.svg",
"ethernet_hub": ":/symbols/affinity/circle/red/hub.svg",
"frame_relay_switch": ":/symbols/affinity/circle/red/isdn.svg",
Expand All @@ -106,7 +106,7 @@
"vmware_guest": ":/symbols/affinity/circle/red/vmware.svg",
"docker_guest": ":/symbols/affinity/circle/red/docker.svg"}

INFINITY_CIRCLE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/gray/cloud.svg",
AFFINITY_CIRCLE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/gray/cloud.svg",
"ethernet_switch": ":/symbols/affinity/circle/gray/switch.svg",
"ethernet_hub": ":/symbols/affinity/circle/gray/hub.svg",
"frame_relay_switch": ":/symbols/affinity/circle/gray/isdn.svg",
Expand All @@ -122,9 +122,9 @@
"docker_guest": ":/symbols/affinity/circle/gray/docker.svg"}

BUILTIN_SYMBOL_THEMES = {"Classic": CLASSIC_SYMBOL_THEME,
"Infinity-square-blue": INFINITY_SQUARE_BLUE_SYMBOL_THEME,
"Infinity-square-red": INFINITY_SQUARE_RED_SYMBOL_THEME,
"Infinity-square-gray": INFINITY_SQUARE_GRAY_SYMBOL_THEME,
"Infinity-circle-blue": INFINITY_CIRCLE_BLUE_SYMBOL_THEME,
"Infinity-circle-red": INFINITY_CIRCLE_RED_SYMBOL_THEME,
"Infinity-circle-gray": INFINITY_CIRCLE_GRAY_SYMBOL_THEME}
"Affinity-square-blue": AFFINITY_SQUARE_BLUE_SYMBOL_THEME,
"Affinity-square-red": AFFINITY_SQUARE_RED_SYMBOL_THEME,
"Affinity-square-gray": AFFINITY_SQUARE_GRAY_SYMBOL_THEME,
"Affinity-circle-blue": AFFINITY_CIRCLE_BLUE_SYMBOL_THEME,
"Affinity-circle-red": AFFINITY_CIRCLE_RED_SYMBOL_THEME,
"Affinity-circle-gray": AFFINITY_CIRCLE_GRAY_SYMBOL_THEME}
32 changes: 20 additions & 12 deletions gns3server/controller/symbols.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,11 @@ def list(self):
if filename.startswith('.'):
continue
symbol_file = posixpath.normpath(os.path.relpath(os.path.join(root, filename), get_resource("symbols"))).replace('\\', '/')
theme = posixpath.dirname(symbol_file).replace('/', '-').capitalize()
symbol_id = ':/symbols/' + symbol_file
symbols.append({'symbol_id': symbol_id,
'filename': symbol_file,
'filename': filename,
'theme': theme,
'builtin': True})
self._symbols_path[symbol_id] = os.path.join(root, filename)

Expand All @@ -81,8 +83,9 @@ def list(self):
continue
symbol_file = posixpath.normpath(os.path.relpath(os.path.join(root, filename), directory)).replace('\\', '/')
symbols.append({'symbol_id': symbol_file,
'filename': symbol_file,
'builtin': False,})
'filename': filename,
'builtin': False,
'theme': "Custom symbols"})
self._symbols_path[symbol_file] = os.path.join(root, filename)

symbols.sort(key=lambda x: x["filename"])
Expand All @@ -99,21 +102,26 @@ def symbols_path(self):
return directory

def get_path(self, symbol_id):
symbol_filename = os.path.splitext(os.path.basename(symbol_id))[0]
theme = self._themes.get(self._current_theme, {})
if not theme:
log.error("Could not find symbol theme '{}'".format(self._current_theme))
#symbol_filename = os.path.splitext(os.path.basename(symbol_id))[0]
#theme = self._themes.get(self._current_theme, {})
#if not theme:
# log.error("Could not find symbol theme '{}'".format(self._current_theme))
try:
return self._symbols_path[theme.get(symbol_filename, symbol_id)]
return self._symbols_path[symbol_id]
#return self._symbols_path[theme.get(symbol_filename, symbol_id)]
except KeyError:
# Symbol not found, let's refresh the cache
try:
self.list()
return self._symbols_path[symbol_id]
except (OSError, KeyError):
log.warning("Could not retrieve symbol '{}'".format(symbol_id))
symbols_path = self._symbols_path
return symbols_path.get(":/symbols/classic/{}".format(os.path.basename(symbol_id)), symbols_path[":/symbols/classic/computer.svg"])
# try to return a symbol with the same name from the classic theme
symbol = self._symbols_path.get(":/symbols/classic/{}".format(os.path.basename(symbol_id)))
if symbol:
return symbol
else:
# return the default computer symbol
log.warning("Could not retrieve symbol '{}'".format(symbol_id))
return self._symbols_path[":/symbols/classic/computer.svg"]

def get_size(self, symbol_id):
try:
Expand Down
2 changes: 2 additions & 0 deletions gns3server/handlers/api/controller/template_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ def create(request, response):

controller = Controller.instance()
template = controller.template_manager.add_template(request.json)
# Reset the symbol list
controller.symbols.list()
response.set_status(201)
response.json(template)

Expand Down
9 changes: 4 additions & 5 deletions tests/controller/test_node.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,16 +259,15 @@ def test_symbol(node, symbols_dir, controller):
Change symbol should change the node size
"""

controller.symbols.theme = "Classic"
node.symbol = ":/symbols/dslam.svg"
assert node.symbol == ":/symbols/dslam.svg"
node.symbol = ":/symbols/classic/dslam.svg"
assert node.symbol == ":/symbols/classic/dslam.svg"
assert node.width == 50
assert node.height == 53
assert node.label["x"] is None
assert node.label["y"] == -40

node.symbol = ":/symbols/cloud.svg"
assert node.symbol == ":/symbols/cloud.svg"
node.symbol = ":/symbols/classic/cloud.svg"
assert node.symbol == ":/symbols/classic/cloud.svg"
assert node.width == 159
assert node.height == 71
assert node.label["x"] is None
Expand Down
8 changes: 5 additions & 3 deletions tests/controller/test_symbols.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,15 @@ def test_list(symbols_dir):
symbols = Symbols()
assert {
'symbol_id': ':/symbols/classic/firewall.svg',
'filename': 'classic/firewall.svg',
'builtin': True
'filename': 'firewall.svg',
'builtin': True,
'theme': 'Classic'
} in symbols.list()
assert {
'symbol_id': 'linux.svg',
'filename': 'linux.svg',
'builtin': False
'builtin': False,
'theme': 'Custom symbols'
} in symbols.list()


Expand Down
5 changes: 3 additions & 2 deletions tests/handlers/api/controller/test_symbol.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ def test_symbols(http_controller):
assert response.status == 200
assert {
'symbol_id': ':/symbols/classic/firewall.svg',
'filename': 'classic/firewall.svg',
'builtin': True
'filename': 'firewall.svg',
'builtin': True,
'theme': 'Classic'
} in response.json


Expand Down

0 comments on commit bae3fb8

Please sign in to comment.