Initial import

Can create a repository, import XML for new releases, and generate signed
feeds. Does not yet handle archives, uploading, etc.

0repo.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2013, Thomas Leonard
+# See the README file for details, or visit http://0install.net.
+import sys
+from repo import cmd
+from zeroinstall import SafeException
+
+version = '0.1'
+
+try:
+	cmd.main(sys.argv)
+except SafeException as ex:
+	print(ex)
+	sys.exit(1)

0repo.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" ?>
+<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface">
+  <name>0repo</name>
+  <summary>manage a repository of 0install feeds</summary>
+  <description>
+    0repo allows a group of developers to publish a set of feeds. It can be
+    used by a single developer, generating a set of static files to publish on
+    a web-server, or as a service which accepts signed updates from a group
+    of developers.
+  </description>
+  <homepage>http://0install.net/0repo.html</homepage>
+
+  <release:management xmlns:release="http://zero-install.sourceforge.net/2007/namespaces/0release">
+    <!-- Update the copy of the version number in the code -->
+    <release:action phase="commit-release">sed -i &quot;s/^version = '.*'$/version = '$RELEASE_VERSION'/&quot; 0repo.py</release:action>
+  </release:management>
+
+  <feed-for interface="http://0install.net/tools/0repo.xml"/>
+
+  <group license="OSI Approved :: GNU Lesser General Public License (LGPL)">
+    <command name="run" path="0repo.py">
+      <runner interface="http://repo.roscidus.com/python/python"/>
+    </command>
+
+    <command name="test" path="tests/test0repo.py">
+      <runner interface="http://repo.roscidus.com/python/python"/>
+      <executable-in-path name="0repo"/>
+    </command>
+
+    <requires interface="http://repo.roscidus.com/python/python">
+      <version not-before="2.7" before="3"/>
+    </requires>
+
+    <requires interface="http://0install.net/2006/interfaces/0publish">
+      <executable-in-var name='ZEROPUBLISH'/>
+    </requires>
+
+    <requires interface="http://0install.net/2007/interfaces/ZeroInstall.xml">
+      <environment insert="" mode="prepend" name="PYTHONPATH"/>
+      <version not-before="2.0"/>
+    </requires>
+
+    <implementation id="." version="0.1-pre"/>
+  </group>
+</interface>

README.md

@@ -0,0 +1,203 @@
+0repo
+=====
+
+Copyright Thomas Leonard, 2013
+
+
+WARNING: much of this isn't implemented yet!
+
+
+Introduction
+------------
+
+The 0repo software provides an easy and reliable way to maintain a repository
+of 0install software for others to use. It can be run interactively by a single
+developer on their laptop to maintain a repository of their own programs, or
+installed as a service to allow a group of people to manage a set of programs
+together.
+
+Developers place new software releases in an "incoming" directory. 0repo
+performs various checks on the new release and, if it's OK, adds it to the
+repository. 0repo signs the published feeds with its GPG key.
+
+The generated files may be rsync'd to a plain web host, without the need for
+special software on the hosting platform.
+
+Features:
+
+- Can be run automatically as part of a scripted release process (for
+  single-developer use).
+
+- Can run as a service to accept contributions from multiple developers.
+
+- Keeps feeds under version control.
+
+- Repositories are always consistent (no missing keys, missing stylesheets,
+  invalid URIs, etc).
+
+- Files can be hosted on a standard web host (e.g. Apache).
+
+- Provides a catalogue file listing all published feeds, which can be polled
+  automatically by mirror sites (e.g. 0mirror).
+
+- Supports both archives hosted within the repository and archives hosted
+  externally.
+
+
+Setup
+-----
+
+Run "0repo create DIR" to create a new repository (directory DIR will be
+created to hold the files and will be populated with an initial configuration).
+
+    $ 0repo create ~/repo
+    $ cd ~/repo
+
+Within this directory you will find:
+
+- 0repo-config.py - configuration settings
+- feeds	        - directory of (unsigned) feeds, initially empty
+- feeds/.git	- version control Git repository for the feeds
+- public	        - output directory (to be rsync'd to hosting provider)
+
+Edit 0repo-config.py and set the required parameters:
+
+These are required:
+
+- The base URL for the feeds
+- The base URL for the archives
+- GPG key to use for signing feeds
+
+These are optional:
+
+- Command to upload feeds to web hosting
+- Command to upload archives to archive hosting
+- GPG keys of trusted contributors
+
+
+Adding a release
+----------------
+
+0repo is designed to be called by other tools, such as 0release, 0template,
+0downstream, etc. However, this section explains how the process can be
+performed manually instead.
+
+Place the XML of your new release in the "incoming" directory. For testing,
+you could add this file as repo/incoming/GNU-Hello-1.3.xml:
+
+    <?xml version="1.0" ?>
+    <interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface"
+               xmlns:compile="http://zero-install.sourceforge.net/2006/namespaces/0compile">
+      <name>GNU Hello</name>
+      <summary>produces a familiar, friendly greeting</summary>
+      <description>The GNU Hello program produces a familiar, friendly greeting.</description>
+      <homepage>http://www.gnu.org/software/hello/</homepage>
+
+      <feed-for interface='{REPO_BASE}/GNU-Hello.xml'/>
+
+      <group arch="*-src">
+        <command name='compile'
+	         shell-command='"$SRCDIR/configure" --prefix="$DISTDIR" &amp;&amp; make install'>
+          <compile:implementation main="bin/hello"/>
+        </command>
+        <requires interface="http://repo.roscidus.com/devel/make">
+          <environment insert="bin" name="PATH"/>
+        </requires>
+        <implementation id="sha1=2aae32fd27d194167eac7eb611d7ce0983f83dd7" version="1.3">
+          <archive extract="hello-1.3" href="http://ftp.gnu.org/gnu/hello/hello-1.3.tar.gz" size="87942"/>
+        </implementation>
+      </group>
+    </interface>
+
+Replace the `<feed-for>`'s `{REPO_BASE}` with the URL of the directory where
+you will publish.
+
+Note that, in this example, the archive is hosted outside of the repository. To
+store the release in the repository, use a relative href on the `<archive>`
+element and place the archive in the incoming directory too. e.g.
+
+    <archive extract="hello-1.3" href="hello-1.3.tar.gz" size="87942"/>
+
+When you are ready, run 0repo inside the "repo" directory. If your new version
+is accepted, a new unsigned feed file be created as repo/feeds/GNU-Hello.xml
+and committed to Git. A signed version of this feed will appear as
+repo/public/GNU-Hello.xml. If you specified a relative URL for the archive, the
+signed version will have an absolute URL and the archive will be copied to your
+configured archives directory. On success, the files are deleted from the
+incoming directory.
+
+You will also have a public/catalog.xml file listing the new program.
+
+When 0repo generates the signed feeds it will also:
+
+- check that each feed's URI is correct for its location
+- add the stylesheet declaration
+- for each relative <archive>'s href, check that the archive exists
+  locally and make the URL absolute
+
+The results go in a separate 'public' directory, which can then be
+transferred to the hosting provider (e.g. using rsync). This directory
+also contains a generated `catalog.xml` file, which 0mirror can poll.
+
+
+Editing feeds
+-------------
+
+You can edit the unsigned feeds under repo/feeds whenever you want. Running
+0repo again will regenerate the signed feeds in repo/public (if the source feed
+has changed). You should commit your changes with `git commit`.
+
+To remove a feed, `git rm repo/feeds/FEED.xml` and run `0repo` again.
+
+
+Running a shared repository
+---------------------------
+
+For a shared repository: the release tool generates the archives and
+the XML for the new version, signs the XML with the developer's key,
+and uploads to a queue (could be e.g. FTP). 0repo downloads the
+contents of the queue to its incoming directory, checks the signature
+and merges the new XML into the feed. If there's a problem, it emails
+the user.
+
+For other edits (e.g. adding a <package-implementation> or adding a missing
+dependency to an already-released version), the contributor sends a Git pull
+request. The repository owner merges the pull request and runs
+0repo.
+
+
+Auditing
+--------
+
+When 0repo adds a new release, the Git commit message includes the XML,
+including the signature, if any. This makes it possible to tell whether a
+malicious update was caused by a compromised 0repo (commit is invalid) or by a
+compromised contributor (the malicious XML is correctly signed by that
+contributor).
+
+Commits made by 0repo are signed with its GPG key. You can check these
+signatures using `git log --show-signature` in the `feeds` directory.
+
+
+Conditions
+----------
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+
+
+Bug Reports
+-----------
+
+Please report any bugs to [the 0install mailing list](http://0install.net/support.html).

repo/build.py

@@ -0,0 +1,90 @@
+# Copyright (C) 2013, Thomas Leonard
+# See the README file for details, or visit http://0install.net.
+
+from __future__ import print_function
+
+import os, subprocess, sys
+from os.path import join, dirname, relpath
+from xml.dom import minidom
+import base64
+
+from zeroinstall.support import xmltools
+from zeroinstall import SafeException
+
+from repo import paths
+
+feed_header = """<?xml version="1.0" ?>
+<?xml-stylesheet type='text/xsl' href='interface.xsl'?>
+"""
+
+def sign_xml(config, source_xml):
+	child = subprocess.Popen(['gpg', '--detach-sign', '--default-key', config.GPG_SIGNING_KEY, '--use-agent', '--output', '-', '-'],
+			stdin = subprocess.PIPE,
+			stdout = subprocess.PIPE,
+			stderr = subprocess.PIPE)
+	stdout, stderr = child.communicate(source_xml)
+	exit_status = child.wait()
+	if exit_status:
+		raise SafeException("Error signing feed: %s" % stderr)
+	if stderr:
+		print(stderr, file=sys.stderr)
+
+	encoded = base64.encodestring(stdout)
+	sig = "\n<!-- Base64 Signature\n" + encoded + "\n-->\n"
+	return source_xml + sig
+
+def generate_public_xml(config, source_xml_path):
+	"""Load source_xml_path and expand any relative URLs."""
+	with open(source_xml_path, 'rb') as stream:
+		doc = minidom.parse(stream)
+
+	root = doc.documentElement
+	declared_iface = root.getAttribute('uri')
+	if not declared_iface:
+		raise SafeException("Feed '{path}' missing 'uri' attribute on root".format(path = source_xml_path))
+
+	if not declared_iface.startswith(config.REPOSITORY_BASE_URL):
+		raise SafeException("Feed '{path}' declares uri='{uri}', which is not under REPOSITORY_BASE_URL ({base})".format(
+			path = source_xml_path,
+			uri = declared_iface,
+			base = config.REPOSITORY_BASE_URL))
+	rel_uri = declared_iface[len(config.REPOSITORY_BASE_URL):]
+
+	expected_path = join('feeds', config.get_feeds_rel_path(rel_uri))
+	if expected_path != source_xml_path:
+		raise SafeException("Feed '{path}' with uri='{uri}' should be located at '{expected_path}'".format(
+			path = source_xml_path,
+			uri = declared_iface,
+			expected_path = expected_path))
+
+	return doc
+
+def build_public_feeds(config):
+	feeds = []
+	for dirpath, dirnames, filenames in os.walk('feeds'):
+		for f in filenames:
+			if f.endswith('.xml') and not f.startswith('.'):
+				source_path = join(dirpath, f)
+				target_path = join("public", paths.get_public_rel_path(config, relpath(source_path, 'feeds')))
+				new_doc = generate_public_xml(config, source_path)
+				if os.path.exists(target_path):
+					with open(target_path, 'rb') as stream:
+						old_doc = minidom.parse(stream)
+					if xmltools.nodes_equal(old_doc.documentElement, new_doc.documentElement):
+						print("%s unchanged" % source_path)
+						continue
+				feeds.append((target_path, new_doc))
+
+	for target_path, new_doc in feeds:
+		new_xml = feed_header + new_doc.documentElement.toxml('utf-8')
+
+		signed_xml = sign_xml(config, new_xml)
+
+		target_dir = dirname(target_path)
+		if not os.path.isdir(target_dir):
+			os.makedirs(target_dir)
+
+		with open(target_path + '.new', 'wb') as stream:
+			stream.write(signed_xml)
+		os.rename(target_path + '.new', target_path)
+		print("Updated", target_path)