Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Fixing imports using --available-as #145

Merged
merged 5 commits into from

1 participant

@jimi-c
Owner

Created a white list file for rsync (/etc/cobbler/import_rsync_whitelist) which allows for fine-grained control over what gets pulled down, and a few minor fixes in the import modules.

Todo: Debian/Ubuntu are not well supported right now, and FreeBSD has no support currently. This should be able to be fixed pretty easily by updating the whitelist file to include things that are required for correct distro detection and setup.

This is probably not ready for 2.2.3, as it requires a lot more testing (which has currently been limited to seeing if the distro is imported mostly correctly).

jimi-c added some commits
@jimi-c jimi-c Fix for import when using --available-as
- currently rsyncs full remote tree, changing that to only import files in a white list
- some modifications to import modules to clean some things up and make available-as work better
- fix in utils.py for path_tail, which was not working right and appending the full path

Todo: Still a lot of things to cleanup to make this work flawlessly. Debian/Ubuntu are particularly problematic due to the way they store every supported version in their mirrors in a sub-tree beyond the pool/ directory we need to scan
2993004
@jimi-c jimi-c Merge branch 'master' into issue_138
b921ceb
@jimi-c jimi-c Some fixups for suse using --available-as
92d6f23
@jimi-c jimi-c Another minor fix for suse imports
- fixing up name when using --available-as (already done in other import modules)
- allowing multiple arch imports (also already done in other imports)
ccac5ff
@jimi-c jimi-c Adding an example line for redhat imports to the whitelist file
74082d3
@jimi-c jimi-c merged commit 9befd7b into from
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Apr 21, 2012
  1. @jimi-c

    Fix for import when using --available-as

    jimi-c authored
    - currently rsyncs full remote tree, changing that to only import files in a white list
    - some modifications to import modules to clean some things up and make available-as work better
    - fix in utils.py for path_tail, which was not working right and appending the full path
    
    Todo: Still a lot of things to cleanup to make this work flawlessly. Debian/Ubuntu are particularly problematic due to the way they store every supported version in their mirrors in a sub-tree beyond the pool/ directory we need to scan
  2. @jimi-c
  3. @jimi-c
  4. @jimi-c

    Another minor fix for suse imports

    jimi-c authored
    - fixing up name when using --available-as (already done in other import modules)
    - allowing multiple arch imports (also already done in other imports)
  5. @jimi-c
This page is out of date. Refresh to see the latest.
View
8 cobbler/api.py
@@ -69,7 +69,7 @@
DEBUG = 5
# FIXME: add --quiet depending on if not --verbose?
-RSYNC_CMD = "rsync -a %s '%s' %s --exclude-from=/etc/cobbler/rsync.exclude --progress"
+RSYNC_CMD = "rsync -aL %s '%s' %s --progress"
# notes on locking:
# BootAPI is a singleton object
@@ -821,6 +821,12 @@ def import_tree(self,mirror_url,mirror_name,network_root=None,kickstart_file=Non
if rsync_flags:
rsync_cmd = rsync_cmd + " " + rsync_flags
+ # if --available-as was specified, limit the files we
+ # pull down via rsync to just those that are critical
+ # to detecting what the distro is
+ if network_root is not None:
+ rsync_cmd = rsync_cmd + " --include-from=/etc/cobbler/import_rsync_whitelist"
+
# kick off the rsync now
utils.run_this(rsync_cmd, (spacer, mirror_url, path), self.logger)
View
1  cobbler/modules/manage_import_debian_ubuntu.py
@@ -84,6 +84,7 @@ def what(self):
def check_for_signature(self,path,cli_breed):
signatures = [
'pool',
+ 'main/debian-installer',
]
#self.logger.info("scanning %s for a debian/ubuntu distro signature" % path)
View
35 cobbler/modules/manage_import_redhat.py
@@ -202,39 +202,6 @@ def get_release_files(self):
data2.append(x)
return data2
- def get_tree_location(self, distro):
- """
- Once a distribution is identified, find the part of the distribution
- that has the URL in it that we want to use for kickstarting the
- distribution, and create a ksmeta variable $tree that contains this.
- """
-
- base = self.get_rootdir()
-
- if self.network_root is None:
- dest_link = os.path.join(self.settings.webdir, "links", distro.name)
- # create the links directory only if we are mirroring because with
- # SELinux Apache can't symlink to NFS (without some doing)
- if not os.path.exists(dest_link):
- try:
- os.symlink(base, dest_link)
- except:
- # this shouldn't happen but I've seen it ... debug ...
- self.logger.warning("symlink creation failed: %(base)s, %(dest)s") % { "base" : base, "dest" : dest_link }
- # how we set the tree depends on whether an explicit network_root was specified
- tree = "http://@@http_server@@/cblr/links/%s" % (distro.name)
- self.set_install_tree(distro, tree)
- else:
- # where we assign the kickstart source is relative to our current directory
- # and the input start directory in the crawl. We find the path segments
- # between and tack them on the network source path to find the explicit
- # network path to the distro that Anaconda can digest.
- tail = utils.path_tail(self.path, base)
- tree = self.network_root[:-1] + tail
- self.set_install_tree(distro, tree)
-
- return
-
def repo_finder(self, distros_added):
"""
This routine looks through all distributions and tries to find
@@ -372,8 +339,6 @@ def distro_adder(self,distros_added,dirname,fnames):
pae_initrd = None
pae_kernel = None
- print "in distro_adder(), directory is %s" % dirname
-
for x in fnames:
adtls = []
View
10 cobbler/modules/manage_import_suse.py
@@ -200,7 +200,7 @@ def get_tree_location(self, distro):
# and the input start directory in the crawl. We find the path segments
# between and tack them on the network source path to find the explicit
# network path to the distro that Anaconda can digest.
- tail = self.path_tail(self.path, base)
+ tail = utils.path_tail(self.path, base)
tree = self.network_root[:-1] + tail
self.set_install_tree(distro, tree)
@@ -328,9 +328,9 @@ def add_entry(self,dirname,kernel,initrd):
archs = [ proposed_arch ]
if len(archs)>1:
- if self.breed in [ "suse" ]:
- self.logger.warning("directory %s holds multiple arches : %s" % (dirname, archs))
- return
+ #if self.breed in [ "suse" ]:
+ # self.logger.warning("directory %s holds multiple arches : %s" % (dirname, archs))
+ # return
self.logger.warning("- Warning : Multiple archs found : %s" % (archs))
distros_added = []
@@ -431,7 +431,7 @@ def get_proposed_name(self,dirname,kernel=None):
"""
if self.network_root is not None:
- name = self.name + "-".join(self.path_tail(os.path.dirname(self.path),dirname).split("/"))
+ name = self.name
else:
# remove the part that says /var/www/cobbler/ks_mirror/name
name = "-".join(dirname.split("/")[5:])
View
4 cobbler/utils.py
@@ -1295,8 +1295,8 @@ def path_tail(apath, bpath):
position = bpath.find(apath)
if position != 0:
return ""
- #rposition = position + len(mirror)
- result = bpath[position:]
+ rposition = position + len(apath)
+ result = bpath[rposition:]
if not result.startswith("/"):
result = "/" + result
return result
View
75 config/import_rsync_whitelist
@@ -0,0 +1,75 @@
+#-----------------------------------------------
+# RHEL/CentOS/SciLinux
+# Ex: cobbler import \
+# --name=fedora-16-remote \
+# --path=rsync://mirrors.kernel.org/mirrors/fedora/releases/16/Fedora/x86_64/os/ \
+# --available-as=http://mirrors.kernel.org/fedora/releases/16/Fedora/x86_64/os/
+#-----------------------------------------------
++ RedHat/
++ RedHat/RPMS/
++ RedHat/rpms/
++ RedHat/Base/
++ Fedora/
++ Fedora/RPMS/
++ Fedora/rpms/
++ CentOS/
++ CentOS/RPMS/
++ CentOS/rpms/
++ CentOS/
++ Packages/
++ Server/
++ Client/
++ SL/
++ images/
++ images/pxeboot/
++ images/pxeboot/*
++ isolinux/
++ isolinux/*
++ */*-release*
+- */kernel-debug*.rpm
+- */kernel-devel*.rpm
+- */kernel-doc*.rpm
+- */kernel-headers*.rpm
++ */kernel-*.rpm
+
+#-----------------------------------------------
+# Debian/Ubuntu
+#-----------------------------------------------
++ pool/
++ dists/
++ dists/*/
++ main/
++ main/debian-installer/
++ main/installer*/
++ main/installer*/current/
++ main/installer*/current/images/
+# all of these should be under the current/images directory...
++ netboot/
++ netboot/ubuntu-installer/
++ netboot/ubuntu-installer/amd64/
++ netboot/ubuntu-installer/i386/
++ netboot/ubuntu-installer/*/initrd*
++ netboot/ubuntu-installer/*/linu*
+
+#-----------------------------------------------
+# SuSE
+# Ex: cobbler import \
+# --name=suse-11.4-remote \
+# --path=rsync://mirrors.kernel.org/mirrors/opensuse/distribution/11.4/repo/oss/ \
+# --available-as=http://mirrors.kernel.org/opensuse/distribution/11.4/repo/oss/
+#-----------------------------------------------
++ boot/
++ boot/i386/
++ boot/i386/loader/
++ boot/i386/loader/initrd
++ boot/i386/loader/linux
++ boot/x86_64/
++ boot/x86_64/loader/
++ boot/x86_64/loader/initrd
++ boot/x86_64/loader/linux
++ suse/
+
+#-----------------------------------------------
+# Exclude everything else
+#-----------------------------------------------
+- *
Something went wrong with that request. Please try again.