diff --git a/examples/gitserver/Dockerfile b/examples/gitserver/Dockerfile new file mode 100644 index 000000000000..e971f2c00082 --- /dev/null +++ b/examples/gitserver/Dockerfile @@ -0,0 +1,13 @@ +# +# This is an example Git server for OpenShift Origin. +# +# The standard name for this image is openshift/origin-gitserver +# +FROM openshift/origin + +ADD hooks/ /var/lib/git-hooks/ +RUN ln -s /usr/bin/openshift /usr/bin/openshift-gitserver && \ + mkdir -p /var/lib/git +VOLUME /var/lib/git + +ENTRYPOINT ["/usr/bin/openshift-gitserver"] diff --git a/examples/gitserver/README.md b/examples/gitserver/README.md new file mode 100644 index 000000000000..8fbfdfbbfda9 --- /dev/null +++ b/examples/gitserver/README.md @@ -0,0 +1,16 @@ +Configurable Git Server +======================= + +This example provides automatic mirroring of Git repositories, intended +for use within a container or Kubernetes pod. It can clone repositories +from remote systems on startup as well as remotely register hooks. It +can automatically initialize and receive Git directories on push. + +In the more advanced modes, it can integrate with an OpenShift server to +automatically perform actions when new repositories are created, like +reading the build configs in the current namespace and performing +automatic mirroring of their input, and creating new build-configs when +content is pushed. + +The Dockerfile built by this example is published as +openshift/origin-gitserver \ No newline at end of file diff --git a/examples/gitserver/hooks/detect-language b/examples/gitserver/hooks/detect-language new file mode 100755 index 000000000000..cbbb72851d5f --- /dev/null +++ b/examples/gitserver/hooks/detect-language @@ -0,0 +1,48 @@ +#!/bin/bash +# +# detect-language returns a string indicating the image repository to use as a base +# image for this source repository. Return "docker.io/*/*" for Docker images, a two +# segment entry for a local image repository, or a single segment name to search +# in the current namespace. Set a tag to qualify the version - e.g. "ruby:1.9.3", +# "nodejs:0.10". +# + +set -o errexit +set -o nounset +set -o pipefail + +function has { + [[ -n $(git ls-tree --full-name --name-only HEAD ${@:1}) ]] +} +function key { + git config --local --get "${1}" +} + +prefix=${PREFIX:-openshift/} + +if has Gemfile; then + echo "${prefix}ruby" + exit 0 +fi + +if has requirements.txt; then + echo "${prefix}python" + exit 0 +fi + +if has package.json app.json; then + echo "${prefix}nodejs" + exit 0 +fi + +if has '*.go'; then + echo "${prefix}golang" + exit 0 +fi + +if has index.php; then + echo "${prefix}php" + exit 0 +fi + +exit 1 \ No newline at end of file diff --git a/examples/gitserver/hooks/post-receive b/examples/gitserver/hooks/post-receive new file mode 100755 index 000000000000..f52d500e6186 --- /dev/null +++ b/examples/gitserver/hooks/post-receive @@ -0,0 +1,54 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +function key { + git config --local --get "${1}" +} +function addkey { + git config --local --add "${1}" "${2}" +} + +function detect { + if detected=$(key openshift.io.detect); then + exit 0 + fi + if ! url=$(key gitserver.self.url); then + echo "detect: no self url set" + exit 0 + fi + + # TODO: make it easier to find the build config name created + # by osc new-app + name=$(basename "${url}") + name="${name%.*}" + + if ! lang=$($(dirname $0)/detect-language); then + exit 0 + fi + echo "detect: found language ${lang} for ${name}" + + if ! osc=$(which osc); then + echo "detect: osc is not installed" + addkey openshift.io.detect 1 + exit 0 + fi + osc new-app "${lang}~${url}" + if webhook=$(osc start-build --list-webhooks="generic" "${name}" | head -n 1); then + addkey openshift.io.webhook "${webhook}" + fi + addkey openshift.io.detect 1 +} + +cat > /tmp/postreceived + +detect + +if webhook=$(key openshift.io.webhook); then + # TODO: print output from the server about the hook status + osc start-build --from-webhook="${webhook}" + # TODO: follow logs + echo "build: started" +fi diff --git a/examples/gitserver/main.go b/examples/gitserver/main.go new file mode 100644 index 000000000000..3dbe254578ab --- /dev/null +++ b/examples/gitserver/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/openshift/origin/pkg/gitserver" +) + +func main() { + if len(os.Args) != 1 { + fmt.Printf(`git-server - Expose Git repositories to the network + +%[1]s`, gitserver.EnvironmentHelp) + os.Exit(0) + } + config, err := gitserver.NewEnviromentConfig() + if err != nil { + log.Fatal(err) + } + log.Fatal(gitserver.Start(config)) +} diff --git a/examples/gitserver/template.yaml b/examples/gitserver/template.yaml new file mode 100644 index 000000000000..3667a13f8e89 --- /dev/null +++ b/examples/gitserver/template.yaml @@ -0,0 +1,74 @@ +apiVersion: v1beta3 +kind: List +items: +- apiVersion: v1beta3 + kind: DeploymentConfig + metadata: + name: gitserver + spec: + triggers: + - type: ConfigChange + replicas: 1 + selector: + run-container: gitserver + template: + metadata: + labels: + run-container: gitserver + spec: + containers: + - name: gitserver + image: openshift/origin-gitserver + ports: + - containerPort: 8080 + env: + - name: PUBLIC_URL + value: "http://gitserver.myproject.local:8080" # TODO: this needs to be resolved from env + - name: GIT_HOME + value: /var/lib/git + - name: HOOK_PATH + value: /var/lib/git-hooks + - name: ALLOW_GIT_PUSH + value: "yes" + - name: ALLOW_GIT_HOOKS + value: "yes" + - name: ALLOW_LAZY_CREATE + value: "yes" + - name: AUTOLINK_CONFIG + value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret + - name: AUTOLINK_NAMESPACE + #value: # TODO: use env generation + - name: AUTOLINK_HOOK + value: + - name: REQUIRE_GIT_AUTH + #value: user:password # if set, authentication is required to push to this server + #- name: GIT_INITIAL_CLONE_1 + # value: + - name: OPENSHIFTCONFIG + value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret + volumeMounts: + - name: config + mountPath: /var/lib/gitsecrets/ + readOnly: true + volumes: + - name: config + secret: + secretName: gitserver-config +- apiVersion: v1beta3 + kind: Secret + metadata: + name: gitserver-config + spec: + data: + # Needs to be populated + admin.kubeconfig: +- apiVersion: v1beta3 + kind: Service + metadata: + name: gitserver + spec: + selector: + run-container: gitserver + ports: + - port: 8080 + targetPort: 8080 diff --git a/hack/build-images.sh b/hack/build-images.sh index 78e9387945b7..f3c6a13d6533 100755 --- a/hack/build-images.sh +++ b/hack/build-images.sh @@ -59,6 +59,7 @@ image openshift/origin-docker-registry images/dockerregistry # images that depend on openshift/origin image openshift/origin-deployer images/deployer image openshift/origin-docker-builder images/builder/docker/docker-builder +image openshift/origin-gitserver examples/gitserver image openshift/origin-sti-builder images/builder/docker/sti-builder # extra images (not part of infrastructure) image openshift/hello-openshift examples/hello-openshift diff --git a/hack/common.sh b/hack/common.sh index ac9ddd8a22ca..a5463ce6cfad 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -55,6 +55,7 @@ readonly OPENSHIFT_BINARY_SYMLINKS=( openshift-deploy openshift-sti-build openshift-docker-build + openshift-gitserver osc osadm ) diff --git a/images/dockerregistry/Dockerfile b/images/dockerregistry/Dockerfile index c7c88acf23f0..9bc08a1e34be 100644 --- a/images/dockerregistry/Dockerfile +++ b/images/dockerregistry/Dockerfile @@ -1,3 +1,9 @@ +# +# This is the integrated OpenShift Origin Docker registry. It is configured to +# publish metadata to OpenShift to provide automatic management of images on push. +# +# The standard name for this image is openshift/origin-docker-registry +# FROM openshift/origin-base ADD config.yml /config.yml diff --git a/pkg/cmd/admin/admin.go b/pkg/cmd/admin/admin.go index 352552cda1a4..b143f138e3ae 100644 --- a/pkg/cmd/admin/admin.go +++ b/pkg/cmd/admin/admin.go @@ -32,7 +32,7 @@ func NewCommandAdmin(name, fullName string, out io.Writer) *cobra.Command { // Main command cmds := &cobra.Command{ Use: name, - Short: "tools for managing an OpenShift cluster", + Short: "Tools for managing an OpenShift cluster", Long: fmt.Sprintf(longDesc), Run: func(c *cobra.Command, args []string) { c.SetOutput(out) diff --git a/pkg/cmd/infra/gitserver/gitserver.go b/pkg/cmd/infra/gitserver/gitserver.go new file mode 100644 index 000000000000..6f4ee0c93253 --- /dev/null +++ b/pkg/cmd/infra/gitserver/gitserver.go @@ -0,0 +1,61 @@ +package gitserver + +import ( + "fmt" + "log" + "net/url" + + cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util" + "github.com/spf13/cobra" + + "github.com/openshift/origin/pkg/gitserver" + "github.com/openshift/origin/pkg/gitserver/autobuild" +) + +const longCommandDesc = ` +Start a Git server + +This command launches a Git HTTP/HTTPS server that supports push and pull, mirroring, +and automatic creation of OpenShift applications on push. + +%[1]s +` + +// NewCommandGitServer launches a Git server +func NewCommandGitServer(name string) *cobra.Command { + cmd := &cobra.Command{ + Use: name, + Short: "Start a Git server", + Long: fmt.Sprintf(longCommandDesc, gitserver.EnvironmentHelp), + Run: func(c *cobra.Command, args []string) { + err := RunGitServer() + cmdutil.CheckErr(err) + }, + } + + return cmd +} + +func RunGitServer() error { + config, err := gitserver.NewEnviromentConfig() + if err != nil { + return err + } + link, err := autobuild.NewAutoLinkBuildsFromEnvironment() + switch { + case err == autobuild.ErrNotEnabled: + case err != nil: + log.Fatal(err) + default: + link.LinkFn = func(name string) *url.URL { return gitserver.RepositoryURL(config, name, nil) } + clones, err := link.Link() + if err != nil { + log.Printf("error: %v", err) + break + } + for name, v := range clones { + config.InitialClones[name] = v + } + } + return gitserver.Start(config) +} diff --git a/pkg/cmd/openshift/openshift.go b/pkg/cmd/openshift/openshift.go index fd554d52de3c..fe8e6a2fea12 100644 --- a/pkg/cmd/openshift/openshift.go +++ b/pkg/cmd/openshift/openshift.go @@ -22,6 +22,7 @@ import ( "github.com/openshift/origin/pkg/cmd/flagtypes" "github.com/openshift/origin/pkg/cmd/infra/builder" "github.com/openshift/origin/pkg/cmd/infra/deployer" + "github.com/openshift/origin/pkg/cmd/infra/gitserver" "github.com/openshift/origin/pkg/cmd/infra/router" "github.com/openshift/origin/pkg/cmd/server/start" "github.com/openshift/origin/pkg/cmd/templates" @@ -63,6 +64,8 @@ func CommandFor(basename string) *cobra.Command { cmd = builder.NewCommandSTIBuilder(basename) case "openshift-docker-build": cmd = builder.NewCommandDockerBuilder(basename) + case "openshift-gitserver": + cmd = gitserver.NewCommandGitServer(basename) case "osc": cmd = cli.NewCommandCLI(basename, basename) case "osadm": @@ -108,6 +111,7 @@ func NewCommandOpenShift() *cobra.Command { deployer.NewCommandDeployer("deploy"), builder.NewCommandSTIBuilder("sti-build"), builder.NewCommandDockerBuilder("docker-build"), + gitserver.NewCommandGitServer("git-server"), ) root.AddCommand(infra) diff --git a/pkg/gitserver/autobuild/autobuild.go b/pkg/gitserver/autobuild/autobuild.go new file mode 100644 index 000000000000..5dba15de8094 --- /dev/null +++ b/pkg/gitserver/autobuild/autobuild.go @@ -0,0 +1,194 @@ +package autobuild + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + kclientcmd "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" + "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors" + + buildapi "github.com/openshift/origin/pkg/build/api" + "github.com/openshift/origin/pkg/client" + "github.com/openshift/origin/pkg/generate/git" + "github.com/openshift/origin/pkg/gitserver" +) + +type AutoLinkBuilds struct { + Namespaces []string + Builders []kapi.ObjectReference + Client client.BuildConfigsNamespacer + + CurrentNamespace string + + PostReceiveHook string + + LinkFn func(name string) *url.URL +} + +var ErrNotEnabled = fmt.Errorf("not enabled") + +func NewAutoLinkBuildsFromEnvironment() (*AutoLinkBuilds, error) { + config := &AutoLinkBuilds{} + + file := os.Getenv("AUTOLINK_CONFIG") + if len(file) == 0 { + return nil, ErrNotEnabled + } + clientConfig, namespace, err := clientFromConfig(file) + if err != nil { + return nil, err + } + client, err := client.New(clientConfig) + if err != nil { + return nil, err + } + config.Client = client + + if value := os.Getenv("AUTOLINK_NAMESPACE"); len(value) > 0 { + namespace = value + } + if len(namespace) == 0 { + return nil, ErrNotEnabled + } + + if value := os.Getenv("AUTOLINK_HOOK"); len(value) > 0 { + abs, err := filepath.Abs(value) + if err != nil { + return nil, err + } + if _, err := os.Stat(abs); err != nil { + return nil, err + } + config.PostReceiveHook = abs + } + + config.Namespaces = []string{namespace} + config.CurrentNamespace = namespace + return config, nil +} + +func clientFromConfig(path string) (*kclient.Config, string, error) { + rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: path} + credentials, err := rules.Load() + if err != nil { + return nil, "", fmt.Errorf("the provided credentials %q could not be loaded: %v", path, err) + } + cfg := kclientcmd.NewDefaultClientConfig(*credentials, &kclientcmd.ConfigOverrides{}) + config, err := cfg.ClientConfig() + if err != nil { + return nil, "", fmt.Errorf("the provided credentials %q could not be used: %v", path, err) + } + namespace, _ := cfg.Namespace() + return config, namespace, nil +} + +func (a *AutoLinkBuilds) Link() (map[string]gitserver.Clone, error) { + errs := []error{} + builders := []*buildapi.BuildConfig{} + for _, namespace := range a.Namespaces { + list, err := a.Client.BuildConfigs(namespace).List(labels.Everything(), fields.Everything()) + if err != nil { + errs = append(errs, err) + continue + } + for i := range list.Items { + builders = append(builders, &list.Items[i]) + } + } + for _, b := range a.Builders { + if hasItem(builders, b) { + continue + } + config, err := a.Client.BuildConfigs(b.Namespace).Get(b.Name) + if err != nil { + errs = append(errs, err) + continue + } + builders = append(builders, config) + } + + hooks := make(map[string]string) + if len(a.PostReceiveHook) > 0 { + hooks["post-receive"] = a.PostReceiveHook + } + + clones := make(map[string]gitserver.Clone) + for _, builder := range builders { + source := builder.Parameters.Source.Git + if source == nil { + continue + } + if builder.Annotations == nil { + builder.Annotations = make(map[string]string) + } + + // calculate the origin URL + uri := source.URI + if value, ok := builder.Annotations["git.openshift.io/origin-url"]; ok { + uri = value + } + if len(uri) == 0 { + continue + } + origin, err := git.ParseRepository(uri) + if err != nil { + errs = append(errs, err) + continue + } + + // calculate the local repository name and self URL + name := builder.Name + if a.CurrentNamespace != builder.Namespace { + name = fmt.Sprintf("%s.%s", builder.Namespace, name) + } + name = fmt.Sprintf("%s.git", name) + self := a.LinkFn(name) + if self == nil { + errs = append(errs, fmt.Errorf("no self URL available, can't update %s", name)) + continue + } + + // we can't clone from ourself + if self.Host == origin.Host { + continue + } + + // update the existing builder + changed := false + if builder.Annotations["git.openshift.io/origin-url"] != origin.String() { + builder.Annotations["git.openshift.io/origin-url"] = origin.String() + changed = true + } + if source.URI != self.String() { + source.URI = self.String() + changed = true + } + if changed { + if _, err := a.Client.BuildConfigs(builder.Namespace).Update(builder); err != nil { + errs = append(errs, err) + continue + } + } + + clones[name] = gitserver.Clone{ + URL: *origin, + Hooks: hooks, + } + } + return clones, errors.NewAggregate(errs) +} + +func hasItem(items []*buildapi.BuildConfig, item kapi.ObjectReference) bool { + for _, c := range items { + if c.Namespace == item.Namespace && c.Name == item.Name { + return true + } + } + return false +} diff --git a/pkg/gitserver/gitserver.go b/pkg/gitserver/gitserver.go new file mode 100644 index 000000000000..c66601549fe2 --- /dev/null +++ b/pkg/gitserver/gitserver.go @@ -0,0 +1,298 @@ +// Package gitserver provides a smart Git HTTP server that can also set and +// remove hooks. The server is lightweight (<7M compiled with a ~2M footprint) +// and can mirror remote repositories in a containerized environment. +package gitserver + +import ( + "fmt" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/AaronO/go-git-http" + "github.com/AaronO/go-git-http/auth" + "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" + "github.com/prometheus/client_golang/prometheus" + + "github.com/openshift/origin/pkg/generate/git" +) + +const ( + initialClonePrefix = "GIT_INITIAL_CLONE_" + EnvironmentHelp = `Supported environment variables: +GIT_HOME + directory containing Git repositories; defaults to current directory +PUBLIC_URL + the url of this server for constructing URLs that point to this repository +GIT_PATH + path to Git binary; defaults to location of 'git' in PATH +HOOK_PATH + path to a directory containing hooks for all repositories; if not set no global hooks will be used +ALLOW_GIT_PUSH + if 'no', pushes will be not be accepted; defaults to true +ALLOW_GIT_HOOKS + if 'no', hooks cannot be read or set; defaults to true +ALLOW_LAZY_CREATE + if 'no', repositories will not automatically be initialized on push; defaults to true +REQUIRE_GIT_AUTH + a user/password combination required to access the repo of the form ":"; defaults to none +GIT_FORCE_CLEAN + if 'yes', any initial repository directories will be deleted prior to start; defaults to no + WARNING: this is destructive and you will lose any data you have already pushed +GIT_INITIAL_CLONE_*=[;] + each environment variable in this pattern will be cloned when the process starts; failures will be logged + must be [A-Z0-9_\-\.], the cloned directory name will be lowercased. If the name is invalid the + process will halt. If the repository already exists on disk, it will be updated from the remote. +` +) + +var ( + invalidCloneNameChars = regexp.MustCompile("[^a-zA-Z0-9_\\-\\.]") + reservedNames = map[string]struct{}{"_": {}} + + eventCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "git_event_count", + Help: "Counter of events broken out for each repository and type", + }, + []string{"repository", "type"}, + ) +) + +func init() { + prometheus.MustRegister(eventCounter) +} + +// Config represents the configuration to use for running the server +type Config struct { + Home string + GitBinary string + URL *url.URL + + AllowHooks bool + AllowPush bool + AllowLazyCreate bool + + HookDirectory string + MaxHookBytes int64 + + Listen string + + AuthenticatorFn func(http http.Handler) http.Handler + + CleanBeforeClone bool + InitialClones map[string]Clone +} + +// Clone is a repository to clone +type Clone struct { + URL url.URL + Hooks map[string]string +} + +// NewDefaultConfig returns a default server config. +func NewDefaultConfig() *Config { + return &Config{ + Home: "", + GitBinary: "git", + Listen: ":8080", + MaxHookBytes: 50 * 1024, + } +} + +// NewEnviromentConfig sets up the initial config from environment variables +func NewEnviromentConfig() (*Config, error) { + config := NewDefaultConfig() + + home := os.Getenv("GIT_HOME") + if len(home) == 0 { + return nil, fmt.Errorf("GIT_HOME is required") + } + abs, err := filepath.Abs(home) + if err != nil { + return nil, fmt.Errorf("Can't make %q absolute: %v", home, err) + } + if stat, err := os.Stat(abs); err != nil || !stat.IsDir() { + return nil, fmt.Errorf("GIT_HOME must be an existing directory: %v", err) + } + config.Home = home + + if publicURL := os.Getenv("PUBLIC_URL"); len(publicURL) > 0 { + valid, err := url.Parse(publicURL) + if err != nil { + return nil, fmt.Errorf("PUBLIC_URL must be a valid URL: %v", err) + } + config.URL = valid + } + + gitpath := os.Getenv("GIT_PATH") + if len(gitpath) == 0 { + path, err := exec.LookPath("git") + if err != nil { + return nil, fmt.Errorf("could not find 'git' in PATH; specify GIT_PATH or set your PATH") + } + gitpath = path + } + config.GitBinary = gitpath + + config.AllowPush = os.Getenv("ALLOW_GIT_PUSH") != "no" + config.AllowHooks = os.Getenv("ALLOW_GIT_HOOKS") != "no" + config.AllowLazyCreate = os.Getenv("ALLOW_LAZY_CREATE") != "no" + + if hookpath := os.Getenv("HOOK_PATH"); len(hookpath) != 0 { + path, err := filepath.Abs(hookpath) + if err != nil { + return nil, fmt.Errorf("HOOK_PATH was set but cannot be made absolute: %v", err) + } + if stat, err := os.Stat(path); err != nil || !stat.IsDir() { + return nil, fmt.Errorf("HOOK_PATH must be an existing directory if set: %v", err) + } + config.HookDirectory = path + } + + if value := os.Getenv("REQUIRE_GIT_AUTH"); len(value) > 0 { + parts := strings.Split(value, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("REQUIRE_GIT_AUTH must be a username and password separated by a ':'") + } + username, password := parts[0], parts[1] + config.AuthenticatorFn = auth.Authenticator(func(info auth.AuthInfo) (bool, error) { + if info.Push && !config.AllowPush { + return false, nil + } + if info.Username != username || info.Password != password { + return false, nil + } + return true, nil + }) + } + + if value := os.Getenv("GIT_LISTEN"); len(value) > 0 { + config.Listen = value + } + + config.CleanBeforeClone = os.Getenv("GIT_FORCE_CLEAN") == "yes" + + clones := make(map[string]Clone) + for _, env := range os.Environ() { + if !strings.HasPrefix(env, initialClonePrefix) { + continue + } + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + continue + } + key, value := parts[0], parts[1] + part := key[len(initialClonePrefix):] + if len(part) == 0 { + continue + } + if len(value) == 0 { + return nil, fmt.Errorf("%s must not have an empty value", key) + } + + defaultName := strings.Replace(strings.ToLower(part), "_", "-", -1) + values := strings.Split(value, ";") + + var uri, name string + switch len(values) { + case 1: + uri, name = values[0], "" + case 2: + uri, name = values[0], values[1] + if len(name) == 0 { + return nil, fmt.Errorf("%s name may not be empty", key) + } + default: + return nil, fmt.Errorf("%s may only have two segments ( or ;)", key) + } + + url, err := git.ParseRepository(uri) + if err != nil { + return nil, fmt.Errorf("%s is not a valid repository URI: %v", key, err) + } + switch url.Scheme { + case "http", "https", "git", "ssh": + default: + return nil, fmt.Errorf("%s %q must be a http, https, git, or ssh URL", key, uri) + } + + if len(name) == 0 { + if n, ok := git.NameFromRepositoryURL(url); ok { + name = n + ".git" + } + } + if len(name) == 0 { + name = defaultName + ".git" + } + + if invalidCloneNameChars.MatchString(name) { + return nil, fmt.Errorf("%s name %q must be only letters, numbers, dashes, or underscores", key, name) + } + if _, ok := reservedNames[name]; ok { + return nil, fmt.Errorf("%s name %q is reserved (%v)", key, name, reservedNames) + } + + clones[name] = Clone{ + URL: *url, + } + } + config.InitialClones = clones + + return config, nil +} + +func handler(config *Config) http.Handler { + git := githttp.New(config.Home) + git.GitBinPath = config.GitBinary + git.UploadPack = config.AllowPush + git.ReceivePack = config.AllowPush + git.EventHandler = func(ev githttp.Event) { + path := ev.Dir + if strings.HasPrefix(path, config.Home+"/") { + path = path[len(config.Home)+1:] + } + eventCounter.WithLabelValues(path, ev.Type.String()).Inc() + } + handler := http.Handler(git) + + if config.AllowLazyCreate { + handler = lazyInitRepositoryHandler(config, handler) + } + + if config.AuthenticatorFn != nil { + handler = config.AuthenticatorFn(handler) + } + return handler +} + +func Start(config *Config) error { + if err := clone(config); err != nil { + return err + } + handler := handler(config) + + ops := http.NewServeMux() + if config.AllowHooks { + ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config)))) + } + /*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL) + io.Copy(os.Stdout, r.Body) + }))*/ + ops.Handle("/metrics", prometheus.UninstrumentedHandler()) + healthz.InstallHandler(ops) + + mux := http.NewServeMux() + mux.Handle("/", prometheus.InstrumentHandler("git", handler)) + mux.Handle("/_/", http.StripPrefix("/_", ops)) + + log.Printf("Serving %s on %s", config.Home, config.Listen) + return http.ListenAndServe(config.Listen, mux) +} diff --git a/pkg/gitserver/hooks.go b/pkg/gitserver/hooks.go new file mode 100644 index 000000000000..a6cae680bf57 --- /dev/null +++ b/pkg/gitserver/hooks.go @@ -0,0 +1,89 @@ +package gitserver + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +func hooksHandler(config *Config) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + segments := strings.Split(r.URL.Path[1:], "/") + for _, s := range segments { + if len(s) == 0 || s == "." || s == ".." { + http.NotFound(w, r) + return + } + } + if !config.AllowPush { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + switch len(segments) { + case 2: + path := filepath.Join(config.Home, segments[0], "hooks", segments[1]) + if segments[0] == "hooks" { + path = filepath.Join(config.HookDirectory, segments[1]) + } + + switch r.Method { + // TODO: support HEAD or prevent GET for security + case "GET": + w.Header().Set("Content-Type", "text/plain") + http.ServeFile(w, r, path) + + case "DELETE": + if err := os.Remove(path); err != nil { + log.Printf("error: attempted to remove %s: %v", path, err) + } + w.WriteHeader(http.StatusNoContent) + + case "PUT": + if stat, err := os.Stat(path); err == nil { + if stat.IsDir() || (stat.Mode()&0111) == 0 { + http.Error(w, fmt.Errorf("only executable hooks can be changed: %v", stat).Error(), http.StatusInternalServerError) + return + } + // unsymlink and overwrite + if (stat.Mode() & os.ModeSymlink) != 0 { + os.Remove(path) + } + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0750) + if err != nil { + http.Error(w, fmt.Errorf("unable to open hook file: %v", err).Error(), http.StatusInternalServerError) + return + } + defer f.Close() + max := config.MaxHookBytes + 1 + body := io.LimitReader(r.Body, max) + buf := make([]byte, max) + n, err := io.ReadFull(body, buf) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + http.Error(w, fmt.Errorf("unable to read hook: %v", err).Error(), http.StatusInternalServerError) + return + } + if int64(n) == max { + http.Error(w, fmt.Errorf("hook was too long, truncated to %d bytes", config.MaxHookBytes).Error(), 422) + } else { + w.WriteHeader(http.StatusOK) + } + if _, err := f.Write(buf[:n]); err != nil { + http.Error(w, fmt.Errorf("unable to write hook: %v", err).Error(), http.StatusInternalServerError) + return + } + + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } + + default: + http.NotFound(w, r) + } + }) +} diff --git a/pkg/gitserver/initializer.go b/pkg/gitserver/initializer.go new file mode 100644 index 000000000000..4f4d1811d887 --- /dev/null +++ b/pkg/gitserver/initializer.go @@ -0,0 +1,213 @@ +package gitserver + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/openshift/origin/pkg/generate/git" +) + +var lazyInitMatch = regexp.MustCompile("^/([^\\/]+?)/info/refs$") + +// lazyInitRepositoryHandler creates a handler that will initialize a Git repository +// if it does not yet exist. +func lazyInitRepositoryHandler(config *Config, handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + handler.ServeHTTP(w, r) + return + } + match := lazyInitMatch.FindStringSubmatch(r.URL.Path) + if match == nil { + handler.ServeHTTP(w, r) + return + } + name := match[1] + if name == "." || name == ".." { + handler.ServeHTTP(w, r) + return + } + path := filepath.Join(config.Home, name) + _, err := os.Stat(path) + if !os.IsNotExist(err) { + handler.ServeHTTP(w, r) + return + } + + self := RepositoryURL(config, name, r) + log.Printf("Lazily initializing bare repository %s", self.String()) + + defaultHooks, err := loadHooks(config.HookDirectory) + if err != nil { + log.Printf("error: unable to load default hooks: %v", err) + http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError) + return + } + + // TODO: capture init hook output for Git + if _, err := newRepository(config, path, defaultHooks, self, nil); err != nil { + log.Printf("error: unable to initialize repo %s: %v", path, err) + http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError) + os.RemoveAll(path) + return + } + eventCounter.WithLabelValues(name, "init").Inc() + + handler.ServeHTTP(w, r) + }) +} + +// RepositoryURL creates the public URL for the named git repo. If both config.URL and +// request are nil, the returned URL will be nil. +func RepositoryURL(config *Config, name string, r *http.Request) *url.URL { + var url url.URL + switch { + case config.URL != nil: + url = *config.URL + case r != nil: + url = *r.URL + url.Host = r.Host + url.Scheme = "http" + default: + return nil + } + url.Path = "/" + name + url.RawQuery = "" + url.Fragment = "" + return &url +} + +func newRepository(config *Config, path string, hooks map[string]string, self *url.URL, origin *url.URL) ([]byte, error) { + var out []byte + repo := git.NewRepositoryForBinary(config.GitBinary) + + if origin != nil { + if err := repo.CloneMirror(path, origin.String()); err != nil { + return out, err + } + } else { + if err := repo.Init(path, true); err != nil { + return out, err + } + } + + if self != nil { + if err := repo.AddLocalConfig(path, "gitserver.self.url", self.String()); err != nil { + return out, err + } + } + + // remove all sample hooks, ignore errors here + if files, err := ioutil.ReadDir(filepath.Join(path, "hooks")); err == nil { + for _, file := range files { + os.Remove(filepath.Join(path, "hooks", file.Name())) + } + } + + for name, hook := range hooks { + dest := filepath.Join(path, "hooks", name) + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return out, err + } + if err := os.Symlink(hook, dest); err != nil { + return out, err + } + } + + if initHook, ok := hooks["init"]; ok { + cmd := exec.Command(initHook) + cmd.Dir = path + result, err := cmd.CombinedOutput() + if err != nil { + return out, fmt.Errorf("init hook failed: %v\n%s", err, string(result)) + } + out = result + } + + return out, nil +} + +// clone clones the provided git repositories +func clone(config *Config) error { + defaultHooks, err := loadHooks(config.HookDirectory) + if err != nil { + return err + } + + errs := []error{} + for name, v := range config.InitialClones { + hooks := mergeHooks(defaultHooks, v.Hooks) + url := v.URL + url.Fragment = "" + path := filepath.Join(config.Home, name) + ok, err := git.IsBareRoot(path) + if err != nil { + errs = append(errs, err) + continue + } + if ok { + if !config.CleanBeforeClone { + continue + } + log.Printf("Removing %s", path) + if err := os.RemoveAll(path); err != nil { + errs = append(errs, err) + continue + } + } + log.Printf("Cloning %s into %s", url.String(), path) + + self := RepositoryURL(config, name, nil) + if _, err := newRepository(config, path, hooks, self, &url); err != nil { + // TODO: tear this directory down + errs = append(errs, err) + continue + } + } + if len(errs) > 0 { + s := []string{} + for _, err := range errs { + s = append(s, err.Error()) + } + return fmt.Errorf("Initial clone failed:\n* %s", strings.Join(s, "\n* ")) + } + return nil +} + +func loadHooks(path string) (map[string]string, error) { + hooks := make(map[string]string) + if len(path) == 0 { + return hooks, nil + } + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + for _, file := range files { + if file.IsDir() || (file.Mode().Perm()&0111) == 0 { + continue + } + hook := filepath.Join(path, file.Name()) + name := filepath.Base(hook) + hooks[name] = hook + } + return hooks, nil +} + +func mergeHooks(hooks ...map[string]string) map[string]string { + hook := make(map[string]string) + for _, m := range hooks { + for k, v := range m { + hook[k] = v + } + } + return hook +}