New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Shared docker volumes with EFS #822

Merged
merged 12 commits into from Jul 14, 2016
Copy path View file
@@ -11,7 +11,14 @@
"BlankRegistryHost": { "Fn::Equals": [ { "Ref": "RegistryHost" }, "" ] },
"Development": { "Fn::Equals": [ { "Ref": "Development" }, "Yes" ] },
"Private": { "Fn::Equals": [ { "Ref": "Private" }, "Yes" ] },
"PrivateApi": { "Fn::Equals": [ { "Ref": "PrivateApi" }, "Yes" ] }
"PrivateApi": { "Fn::Equals": [ { "Ref": "PrivateApi" }, "Yes" ] },
"RegionHasEFS": {
"Fn::Or": [
{ "Fn::Equals": [ { "Ref": "AWS::Region" }, "us-east-1" ]},
{ "Fn::Equals": [ { "Ref": "AWS::Region" }, "us-west-2" ]},
{ "Fn::Equals": [ { "Ref": "AWS::Region" }, "eu-west-1" ]}
]
}
},
"Mappings": {
"RegionConfig": {
@@ -897,6 +904,18 @@
"bootcmd:\n",
" - mkswap /dev/xvdb\n",
" - swapon /dev/xvdb\n",
" - yum install -y nfs-utils\n",
" - mkdir /volumes\n",
{ "Fn::If": [ "RegionHasEFS",
{ "Fn::Join": [ "", [
" - while true; do mount -t nfs -o nfsvers=4.1 $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone).",
{ "Ref": "VolumeFilesystem" },
".efs.",
{ "Ref": "AWS::Region" },
".amazonaws.com:/ /volumes && break; sleep 5; done\n"
] ] },
""
] },
" - [ cloud-init-per, instance, docker_storage_setup, /usr/bin/docker-storage-setup ]\n",
" - echo ECS_CLUSTER=", { "Ref": "Cluster" }, " >> /etc/ecs/ecs.config\n",
" - echo ECS_ENGINE_AUTH_TYPE=docker >> /etc/ecs/ecs.config\n",
@@ -1437,6 +1456,53 @@
"ProvisionedThroughput": { "ReadCapacityUnits": "5", "WriteCapacityUnits": "5" }
}
},
"VolumeFilesystem": {
"Type": "AWS::EFS::FileSystem",
"Condition": "RegionHasEFS",
"Properties": {
"FileSystemTags": [
{ "Key": "Name", "Value": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "shared-volumes" ] ] } }
]
}
},
"VolumeSecurity": {
"Type": "AWS::EC2::SecurityGroup",
"Condition": "RegionHasEFS",
"Properties": {
"GroupDescription": "volume security group",
"SecurityGroupIngress": [
{ "IpProtocol": "tcp", "FromPort": "2049", "ToPort": "2049", "CidrIp": { "Ref": "VPCCIDR" } }
],
"VpcId": { "Ref": "Vpc" }
}
},
"VolumeTarget0": {
"Type": "AWS::EFS::MountTarget",
"Condition": "RegionHasEFS",
"Properties": {
"FileSystemId": { "Ref": "VolumeFilesystem" },
"SubnetId": { "Ref": "Subnet0" },
"SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
}
},
"VolumeTarget1": {
"Type": "AWS::EFS::MountTarget",
"Condition": "RegionHasEFS",
"Properties": {
"FileSystemId": { "Ref": "VolumeFilesystem" },
"SubnetId": { "Ref": "Subnet1" },
"SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
}
},
"VolumeTarget2": {
"Type": "AWS::EFS::MountTarget",
"Condition": "RegionHasEFS",
"Properties": {
"FileSystemId": { "Ref": "VolumeFilesystem" },
"SubnetId": { "Ref": "Subnet2" },
"SecurityGroups": [ { "Ref": "VolumeSecurity" } ]
}
},
"Settings": {
"Properties": {
"AccessControl": "Private",
Copy path View file
@@ -191,7 +191,7 @@ func (m Manifest) Validate() error {
regexValidCronLabel := regexp.MustCompile(`\A[a-zA-Z][-a-zA-Z0-9]{3,29}\z`)

for _, entry := range map[string]ManifestEntry(m) {
labels := entry.labelsByPrefix("convox.cron")
labels := entry.LabelsByPrefix("convox.cron")
for k := range labels {
parts := strings.Split(k, ".")
if len(parts) != 3 {
@@ -998,9 +998,11 @@ func (me ManifestEntry) Label(key string) string {
return ""
}

func (me ManifestEntry) labelsByPrefix(prefix string) map[string]string {
// LabelsByPrefix filters Docker Compose label names by prefixes and returns
// a map of label names to values that match.
func (e ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
returnLabels := make(map[string]string)
switch labels := me.Labels.(type) {
switch labels := e.Labels.(type) {
case map[interface{}]interface{}:
for k, v := range labels {
ks, ok := k.(string)
Copy path View file
@@ -7,6 +7,7 @@ import (
"html/template"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"

@@ -163,9 +164,10 @@ func (m Manifest) AppName() string {
return m[0].app.Name
}

func (me ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
// LabelsByPrefix will return the labels that have a given prefix
func (e ManifestEntry) LabelsByPrefix(prefix string) map[string]string {
returnLabels := make(map[string]string)
switch labels := me.Labels.(type) {
switch labels := e.Labels.(type) {
case map[interface{}]interface{}:
for k, v := range labels {
ks, ok := k.(string)
@@ -468,13 +470,38 @@ func (me ManifestEntry) EnvMap() map[string]string {
return envs
}

func (me ManifestEntry) MountableVolumes() []string {
volumes := []string{}
// MountableVolume describes a mountable volume
type MountableVolume struct {
Host string
Container string
}

// MountableVolumes return the mountable volumes for a manifest entry
func (e ManifestEntry) MountableVolumes() []MountableVolume {
volumes := []MountableVolume{}

for _, volume := range e.Volumes {
parts := strings.Split(volume, ":")

for _, volume := range me.Volumes {
if strings.HasPrefix(volume, "/var/run/docker.sock") {
volumes = append(volumes, volume)
// if only one volume part use it for both sides
if len(parts) == 1 {
parts = append(parts, parts[0])
}

// if we dont have two volume parts bail
if len(parts) != 2 {
continue
}

// only support absolute paths for volume source
if !filepath.IsAbs(parts[0]) {
continue
}

volumes = append(volumes, MountableVolume{
Host: parts[0],
Container: parts[1],
})
}

return volumes
Oops, something went wrong.
ProTip! Use n and p to navigate between commits in a pull request.