Permalink
Browse files

Merge branch 'promote'

  • Loading branch information...
2 parents bb7a9bb + 3605399 commit 0e72ef5ac396af0a5090ab5af85bdfd2288e3b55 @technomancy committed Jul 13, 2012
View
@@ -15,3 +15,4 @@ target/
/.lein-deps-sum
/.lein-failures
+/local-resources
View
@@ -22,7 +22,7 @@ it. Regardless of how you run it, you first need to do some setup:
* Debian: `apt-get install sqlite3`
* Mac OS X Homebrew: `brew install sqlite`
-3. Create an initial sqlite database: `mkdir data; sqlite3 data/dev_db < clojars.sql`
+3. Run the DB migrations: `lein run -m clojars.db.migrate`
To run the application using Leinigen 2:
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>robert</groupId>
+ <artifactId>hooke</artifactId>
+ <version>1.1.2</version>
+ <name>hooke</name>
+ <description>Hooke your functions!</description>
+ <scm>
+ <connection>scm:git:git://github.com/technomancy/robert-hooke.git</connection>
+ <developerConnection>scm:git:ssh://git@github.com/technomancy/robert-hooke.git</developerConnection>
+ <tag>19ce36f7a3b0704cdcde821ebf4b4721ec903efb</tag>
+ <url>https://github.com/technomancy/robert-hooke</url>
+ </scm>
+ <build>
+ <sourceDirectory>src</sourceDirectory>
+ <testSourceDirectory>test</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>test-resources</directory>
+ </testResource>
+ </testResources>
+ </build>
+ <repositories>
+ <repository>
+ <id>central</id>
+ <url>http://repo1.maven.org/maven2</url>
+ </repository>
+ <repository>
+ <id>clojure</id>
+ <url>http://build.clojure.org/releases</url>
+ </repository>
+ <repository>
+ <id>clojure-snapshots</id>
+ <url>http://build.clojure.org/snapshots</url>
+ </repository>
+ <repository>
+ <id>clojars</id>
+ <url>http://clojars.org/repo/</url>
+ </repository>
+ </repositories>
+ <dependencies>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>clojure</artifactId>
+ <version>1.3.0-beta1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>robert</groupId>
+ <artifactId>hooke</artifactId>
+ <packaging>jar</packaging>
+ <version>1.2.0-SNAPSHOT</version>
+ <name>hooke</name>
+ <description>Hooke your functions!</description>
+ <url>https://github.com/technomancy/robert-hooke</url>
+ <licenses>
+ <license>
+ <name>Eclipse Public License</name>
+ <url>http://www.eclipse.org/legal/epl-v10.html</url>
+ </license>
+ </licenses>
+ <scm>
+ <connection>scm:git:git://github.com/technomancy/robert-hooke.git</connection>
+ <developerConnection>scm:git:ssh://git@github.com/technomancy/robert-hooke.git</developerConnection>
+ <tag>727961c5242de367f233c50fd789a0670780e1b9</tag>
+ <url>https://github.com/technomancy/robert-hooke</url>
+ </scm>
+ <build>
+ <sourceDirectory>src</sourceDirectory>
+ <testSourceDirectory>test</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>dev-resources</directory>
+ </testResource>
+ <testResource>
+ <directory>resources</directory>
+ </testResource>
+ </testResources>
+ <directory>target</directory>
+ <outputDirectory>target/classes</outputDirectory>
+ </build>
+ <repositories>
+ <repository>
+ <id>central</id>
+ <url>http://repo1.maven.org/maven2</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ <repository>
+ <id>clojars</id>
+ <url>https://clojars.org/repo/</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ </repositories>
+ <dependencies>
+ <dependency>
+ <groupId>org.clojure</groupId>
+ <artifactId>clojure</artifactId>
+ <version>1.4.0</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
+
+<!-- This file was autogenerated by Leiningen.
+ Please do not edit it directly; instead edit project.clj and regenerate it.
+ It should not be considered canonical data. For more information see
+ https://github.com/technomancy/leiningen -->
View
@@ -4,7 +4,8 @@
[org.apache.maven/maven-model "3.0.4"
:exclusions
[org.codehaus.plexus/plexus-utils]]
- [com.cemerick/pomegranate "0.0.12"]
+ [com.cemerick/pomegranate "0.0.13"]
+ [s3-wagon-private "1.0.0"]
[compojure "1.0.1"]
[ring/ring-jetty-adapter "1.0.2"]
[hiccup "0.3.8"]
@@ -20,7 +21,8 @@
:dependencies [[kerodon "0.0.4"]
[nailgun-shim "0.0.1"]]}
:dev {:dependencies [[kerodon "0.0.4"]
- [nailgun-shim "0.0.1"]]}}
+ [nailgun-shim "0.0.1"]]
+ :resource-paths ["local-resources"]}}
:plugins [[lein-ring "0.6.3"]]
:ring {:handler clojars.web/clojars-app}
:aot [clojars.scp]
View
@@ -196,6 +196,12 @@
(defn update-jar [account {:keys [group name version
description homepage authors]}]
+ (let [[{:keys [promoted_at]}] (select jars (fields :promoted_at)
+ (where {:group_name group
+ :jar_name name
+ :version version}))]
+ (when promoted_at
+ (throw (Exception. "Already promoted."))))
(update jars
(set-fields {:user account
:created (get-time)
View
@@ -0,0 +1,41 @@
+(ns clojars.db.migrate
+ (:require [clojure.java.jdbc :as sql]
+ [clojars.config :refer [config]])
+ (:import (java.sql Timestamp)))
+
+(defn initial-schema []
+ (doseq [cmd (.split (slurp "clojars.sql") ";\n\n")]
+ ;; needs to succeed even if tables exist since this migration
+ ;; hasn't been recorded in extant DBs before migrations were introduced
+ (try (sql/do-commands cmd)
+ (catch java.sql.BatchUpdateException _))))
+
+(defn add-promoted-field []
+ (sql/do-commands "ALTER TABLE jars ADD COLUMN promoted_at DATE"))
+
+;; migrations mechanics
+
+(defn run-and-record [migration]
+ (println "Running migration:" (:name (meta migration)))
+ (migration)
+ (sql/insert-values "migrations" [:name :created_at]
+ [(str (:name (meta migration)))
+ (Timestamp. (System/currentTimeMillis))]))
+
+(defn migrate [& migrations]
+ (sql/with-connection (config :db)
+ (try (sql/create-table "migrations"
+ [:name :varchar "NOT NULL"]
+ [:created_at :timestamp
+ "NOT NULL" "DEFAULT CURRENT_TIMESTAMP"])
+ (catch Exception _))
+ (sql/transaction
+ (let [has-run? (sql/with-query-results run ["SELECT name FROM migrations"]
+ (set (map :name run)))]
+ (doseq [m migrations
+ :when (not (has-run? (str (:name (meta m)))))]
+ (run-and-record m))))))
+
+(defn -main []
+ (migrate #'initial-schema
+ #'add-promoted-field))
View
@@ -2,6 +2,7 @@
(:require [clojars.scp]
[ring.adapter.jetty :refer [run-jetty]]
[clojars.web :refer [clojars-app]]
+ [clojars.promote :as promote]
[clojars.config :refer [config configure]])
(:import com.martiansoftware.nailgun.NGServer
java.net.InetAddress)
@@ -21,6 +22,7 @@
(defn -main [& args]
(configure args)
+ (promote/start)
(start-jetty)
(start-nailgun))
View
@@ -8,6 +8,9 @@
:version (.getVersion model)
:description (.getDescription model)
:homepage (.getUrl model)
+ :url (.getUrl model)
+ :licenses (.getLicenses model)
+ :scm (.getScm model)
:authors (vec (map #(.getName %) (.getContributors model)))
;; TODO: doesn't appear to be used anywhere?
:dependencies (vec (mapcat (fn [d] [(symbol (.getGroupId d)
@@ -21,5 +24,4 @@
(with-open [reader (io/reader file)]
(.read (MavenXpp3Reader.) reader)))
-
(def pom-to-map (comp model-to-map read-pom))
View
@@ -0,0 +1,127 @@
+(ns clojars.promote
+ (:require [clojars.config :refer [config]]
+ [clojars.maven :as maven]
+ [clojure.java.io :as io]
+ [clojure.java.shell :as sh]
+ [cemerick.pomegranate.aether :as aether]
+ [clojars.db :as db]
+ [clojure.java.jdbc :as sql]
+ [korma.core :refer [select fields where update set-fields]])
+ (:import (java.util.concurrent LinkedBlockingQueue)
+ (org.springframework.aws.maven SimpleStorageServiceWagon)))
+
+(defn file-for [group artifact version extension]
+ (let [filename (format "%s-%s.%s" artifact version extension)]
+ (io/file (config :repo) group artifact version filename)))
+
+(defn check-file [blockers file]
+ (if (.exists file)
+ blockers
+ (conj blockers (str "Missing file " (.getName file)))))
+
+(defn check-version [blockers version]
+ (if (re-find #"-SNAPSHOT$" version)
+ (conj blockers "Snapshot versions cannot be promoted")
+ blockers))
+
+(defn check-field [blockers info field]
+ (if (field info)
+ blockers
+ (conj blockers (str "Missing " (name field)))))
+
+(declare check-signature)
+
+(defn- fetch-key [signature err]
+ (if (re-find #"Can't check signature: public key not found" err)
+ (let [key (second (re-find #"using \w+ key ID (.+)" err))
+ {:keys [exit]} (sh/sh "gpg" "--recv-keys" key)]
+ (if (zero? exit)
+ (check-signature signature)))))
+
+(defn- check-signature [signature]
+ (let [err (java.io.StringWriter.)
+ out (java.io.StringWriter.)
+ {:keys [exit]} (binding [*err* (java.io.PrintWriter. err), *out* out]
+ (sh/sh "gpg" "--verify" (str signature)))]
+ (or (zero? exit)
+ (fetch-key signature (str err)))))
+
+(defn signed? [blockers file]
+ (if (check-signature (str file ".asc"))
+ blockers
+ (conj blockers (str file " is not signed."))))
+
+(defn unpromoted? [blockers {:keys [group name version]}]
+ (let [[{:keys [promoted_at]}] (select db/jars (fields :promoted_at)
+ (where {:group_name group
+ :jar_name name
+ :version version}))]
+ (if promoted_at
+ (conj blockers "Already promoted.")
+ blockers)))
+
+(defn blockers [{:keys [group name version]}]
+ (let [jar (file-for group name version "jar")
+ pom (file-for group name version "pom")
+ info (try (maven/pom-to-map pom)
+ (catch Exception _ {}))]
+ ;; TODO: convert this to a lazy seq for cheaper qualification checks
+ (-> []
+ (check-version version)
+ (check-file jar)
+ (check-file pom)
+
+ (check-field info :description)
+ (check-field info :url)
+ (check-field info :licenses)
+ (check-field info :scm)
+
+ (signed? jar)
+ (signed? pom)
+ (unpromoted? info))))
+
+(def releases {:url "s3://clojars/releases/"
+ :username (config :releases-access-key)
+ :passphrase (config :releases-secret-key)})
+
+(aether/register-wagon-factory! "s3" (constantly (SimpleStorageServiceWagon.)))
+
+(defn- add-coords [{:keys [group name version classifier] :as info}
+ files extension]
+ ;; TODO: classifier?
+ (assoc files [(symbol group name) version :extension extension]
+ (file-for group name version extension)))
+
+(defn- deploy-to-s3 [info]
+ (let [files (reduce (partial add-coords info) {}
+ ["jar" "jar.asc" "pom" "pom.asc"])]
+ (aether/deploy-artifacts :artifacts (keys files)
+ :files files
+ :transfer-listener :stdout
+ :repository {"releases" releases})))
+
+(defn promote [{:keys [group name version] :as info}]
+ (sql/with-connection (config :db)
+ (sql/transaction
+ (let [blockers (blockers info)]
+ (if (empty? blockers)
+ (do
+ (println "Promoting" info)
+ (update db/jars
+ (set-fields {:promoted_at (java.util.Date.)})
+ (where {:group_name group :jar_name name :version version}))
+ (deploy-to-s3 info))
+ blockers)))))
+
+(defonce queue (LinkedBlockingQueue.))
+
+(defn start []
+ (.start (Thread. #(loop []
+ (try (promote (.take queue))
+ (catch Exception e
+ (.printStackTrace e)))
+ (recur)))))
+
+;; TODO: probably worth periodically queueing all non-promoted
+;; releases into here to catch things that fall through the cracks,
+;; say if the JVM is restarted before emptying this queue.
Oops, something went wrong.

0 comments on commit 0e72ef5

Please sign in to comment.