diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py
index 9791dbbeb0b184..efec15082d0dc8 100644
--- a/lib/ansible/plugins/inventory/__init__.py
+++ b/lib/ansible/plugins/inventory/__init__.py
@@ -402,7 +402,10 @@ def _add_host_to_keyed_groups(self, keys, variables, host, strict=False):
 
                     if key:
                         prefix = keyed.get('prefix', '')
-                        sep = keyed.get('separator', '_')
+                        if prefix == '':
+                            sep = ''
+                        else:
+                            sep = keyed.get('separator', '_')
                         raw_parent_name = keyed.get('parent_group', None)
                         if raw_parent_name:
                             try: