Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

New AssetSync Class #3

Merged
merged 2 commits into from

2 participants

David Estes Benoit Hediard
David Estes

Working on new assetSync class that can sync more than one provider (if using to sync to multiple regions). Also leverages new ContentType logic and localStorageProvider. This may not be merge ready yet as I havent integrated into the gant scripts yet but thought you might want to have a look.

David Estes

Ok I haven't tested this at all, but the AssetSync class is no longer dependent on s3 specific classes. Supports iterating over multiple providers. I did remove some gant code that should no longer be needed.

Benoit Hediard benorama merged commit 4feb946 into from
Benoit Hediard
Owner

OK, thanks !
I'll have a look on this tonight.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
4 CdnAssetPipelineGrailsPlugin.groovy
View
@@ -8,9 +8,7 @@ class CdnAssetPipelineGrailsPlugin {
def author = "Benoit Hediard"
def authorEmail = "ben@benorama.Ccm"
def title = "CDN Asset Pipeline Plugin"
- def description = '''\
-Adds new asset-cdn-push command to upload assets generated by Asset Pipeline Plugin to cloud CDNs.
-'''
+ def description = 'Adds new asset-cdn-push command to upload assets generated by Asset Pipeline Plugin to cloud CDNs.'
def documentation = "http://github.com/agorapulse/grails-cdn-asset-pipeline"
def license = "APACHE"
2  grails-app/conf/BuildConfig.groovy
View
@@ -18,7 +18,7 @@ grails.project.dependency.resolution = {
export = false
}
- compile ':karman-aws:0.2.0'
+ compile ':karman-aws:0.3.1'
compile ':asset-pipeline:1.6.1'
}
}
54 scripts/AssetCdnPush.groovy
View
@@ -55,56 +55,18 @@ target(main: "Upload static assets to CDN") {
}
assetCompile() // Compile assets
- loadProvider() // Load provider
+ def assetSyncClass = classLoader.loadClass('asset.pipeline.cdn.AssetSync')
+ def options = [:]
+ options.expirationDate = expirationDate
+ options.providers = providers
+ def assetSync = assetSyncClass.newInstance(options,eventListener)
- int uploadCount = 0
+
// Push resources to directory
- def assetPath = 'target/assets'
- def assetDir = new File(assetPath)
- if (!assetDir.exists()) {
- event("StatusError", ["Could not push assets, target/assets directory not found"])
- } else {
- List list = []
- assetDir.eachFileRecurse (FileType.FILES) { file ->
- list << file
- }
-
- int total = list.size()
- list.eachWithIndex { File file, index ->
- String name = prefix + file.path.replace("${assetPath}/", '')
- event("StatusUpdate", ["Uploading File ${index} of ${total} - $name"])
- def cloudFile = provider[directory][name]
- if (expirationDate) {
- cloudFile.setMetaAttribute(Headers.CACHE_CONTROL, "PUBLIC, max-age=${(expirationDate.time / 1000).toInteger()}, must-revalidate")
- cloudFile.setMetaAttribute(Headers.EXPIRES, expirationDate)
- }
- // Specify some content types for extension not handled by URLConnection.guessContentType
- Map contentTypes = [
- css: 'text/css',
- gz: 'application/x-compressed',
- js: 'application/javascript',
- pdf: 'application/pdf',
- eot: 'application/vnd.ms-fontobject',
- otf: 'font/opentype',
- svg: 'image/svg+xml',
- ttf: 'application/x-font-ttf',
- woff: 'application/x-font-woff'
- ]
- String extension = file.name.tokenize('.').last()
- if (contentTypes[extension]) {
- cloudFile.contentType = contentTypes[extension]
- } else {
- cloudFile.contentType = URLConnection.guessContentTypeFromName(file.name)
- }
- cloudFile.bytes = file.bytes
- // Upload file
- cloudFile.save('public-read')
- uploadCount++
- }
- }
+ assetSync.sync()
- event("StatusFinal", ["Assets push complete: $uploadCount assets uploaded to directory '$directory'"])
+ event("StatusFinal", ["Assets push complete!"])
}
setDefaultTarget(main)
56 scripts/_AssetCdn.groovy
View
@@ -1,8 +1,10 @@
includeTargets << grailsScript("_GrailsInit")
+includeTargets << new File(karmanPluginDir, "scripts/_InitKarman.groovy")
includeTargets << new File(assetPipelinePluginDir, "scripts/_AssetCompile.groovy")
target(loadConfig: "Load CDN assets config") {
depends(compile, parseArguments)
+
if (argsMap['help']) {
println USAGE
@@ -11,23 +13,34 @@ target(loadConfig: "Load CDN assets config") {
loadApp()
configureApp()
+ initKarman()
+ providers = []
def cdnAssetsConfig = grailsApp.config.grails.assets?.cdn
- // Parse arguments
- providerName = argsMap['provider'] ?: cdnAssetsConfig?.provider
- directory = argsMap['directory'] ?: cdnAssetsConfig?.directory
- accessKey = argsMap['access-key'] ?: cdnAssetsConfig?.accessKey
- secretKey = argsMap['secret-key'] ?: cdnAssetsConfig?.secretKey
- region = argsMap['region'] ?: cdnAssetsConfig?.region
+ providers = cdnAssetsConfig.providers ?: []
- if (providerName == 'S3') {
+ if(providers.size() == 0) {
+ def providerObject = [:]
+ providerObject.provider = argsMap['provider'] ?: cdnAssetsConfig?.provider
+ providerObject.directory = argsMap['directory'] ?: cdnAssetsConfig?.directory
+ providerObject.accessKey = argsMap['access-key'] ?: cdnAssetsConfig?.accessKey
+ providerObject.secretKey = argsMap['secret-key'] ?: cdnAssetsConfig?.secretKey
+ providerObject.region = argsMap['region'] ?: cdnAssetsConfig?.region
+ providers << providerObject
+ }
+
+ def prefix = argsMap['prefix'] ?: cdnAssetsConfig.prefix
+ providers.each { provider ->
+ if(provider.provider?.toLowerCase() == 's3')
def awsConfig = grailsApp.config.grails.plugin?.awssdk
- if (!directory) directory = awsConfig?.s3?.bucket ?: awsConfig?.bucket
- if (!accessKey) accessKey = awsConfig?.s3?.accessKey ?: awsConfig?.accessKey
- if (!secretKey) secretKey = awsConfig?.s3?.secretKey ?: awsConfig?.secretKey
- if (!region) region = awsConfig?.s3?.region ?: awsConfig?.region ?: ''
+ if (!provider.directory) provider.directory = awsConfig?.s3?.bucket ?: awsConfig?.bucket
+ if (!provider.accessKey) provider.accessKey = awsConfig?.s3?.accessKey ?: awsConfig?.accessKey
+ if (!provider.secretKey) provider.secretKey = awsConfig?.s3?.secretKey ?: awsConfig?.secretKey
+ if (!provider.region) provider.region = awsConfig?.s3?.region ?: awsConfig?.region ?: ''
+ if(!provider.storagePath) provider.storagePath = prefix
}
+
// Global expirationDate var
expirationDate = null
@@ -42,25 +55,4 @@ target(loadConfig: "Load CDN assets config") {
}
}
- // Global prefix var
- prefix = argsMap['prefix'] ?: cdnAssetsConfig.prefix ?: ''
- if (!prefix.endsWith('/')) prefix = "$prefix/"
- if (prefix.startsWith('/')) prefix = prefix.replaceFirst('/', '')
}
-
-target(loadProvider: "Load Karman provider") {
- depends(loadConfig)
-
- // Load provider
- try {
- String className = "com.bertramlabs.plugins.karman.${providerName == 'S3' ? 'aws' : providerName.toLowerCase()}.${providerName}StorageProvider"
- provider = Class.forName(className, false, Thread.currentThread().contextClassLoader).newInstance(
- accessKey: accessKey,
- secretKey: secretKey,
- region: region
- )
- } catch (ClassNotFoundException exception) {
- event("StatusError", ["Provider class not found: ${exception.message}."])
- exit 1
- }
-}
86 src/groovy/asset/pipeline/cdn/AssetSync.groovy
View
@@ -0,0 +1,86 @@
+package asset.pipeline.cdn
+
+import com.bertramlabs.plugins.karman.*
+import com.bertramlabs.plugins.karman.local.*
+
+class AssetSync {
+
+ def providers = []
+ def localProvider
+ def localDirectory = 'assets'
+ def expirationDate
+ def eventListener
+
+ AssetSync(options,eventListener) {
+ this.eventListener = eventListener
+
+ providers = options.providers ?: []
+ if(options.localProvider) {
+ localProvider = localProvider
+ } else {
+ localProvider = StorageProvider.create(provider: 'local', basePath: options.assetPath ?: 'target')
+ }
+ localDirectory = options.localDirectory ?: localDirectory
+ expirationDate = options.expirationDate
+
+ }
+
+ def sync() {
+ if(localProvider[localDirectory].exists() == false) {
+ eventListener?.triggerEvent("StatusError", "Could not push assets, target/assets directory not found")
+ return false
+ }
+ providers.eachWithIndex { provider, index ->
+ eventListener?.triggerEvent("StatusUpdate", "Syncing Assets with Storage Provider ${index+1} of ${providers.size()}")
+
+ if(synchronizeProvider(provider.clone())) {
+ provider.synchronized = true //This flag is marked on the provider map when its synchronized
+ }
+ }
+ }
+
+ def synchronizeProvider(providerMeta) {
+
+ try {
+ def remoteDirectory = providerMeta.remove('storagePath')
+ if (!remoteDirectory.endsWith('/')) {
+ remoteDirectory = "${remoteDirectory}/"
+ }
+ if (remoteDirectory.startsWith('/')) {
+ remoteDirectory = remoteDirectory.replaceFirst('/', '')
+ }
+ def provider = StorageProvider.create(providerMeta + [defaultFileACL: CloudFileACL.PublicRead])
+ def files = localProvider[localDirectory].listFiles()
+ def remoteManifestFile = provider[remoteDirectory ?: localDirectory]['manifest.properties'] //Lets check if a remote manifest exists
+ def remoteManifest = new Properties()
+
+ if(remoteManifestFile.exists()) {
+ remoteManifest.load(remoteManifestFile.inputStream)
+ // def localManifestFile = localProvider[localDirectory]['manifest.properties']
+ // if(localManifestFile.exists()) {
+
+ // }
+ // TODO: We need to download this manifest, run a comparison and only upload/remove whats changed
+ }
+
+ files.eachWithIndex { localFile, index ->
+ eventListener?.triggerEvent("StatusUpdate", "Uploading File ${index+1} of ${files.size()} - ${localFile.name}")
+ def cloudFile = provider[remoteDirectory ?: localDirectory][localFile.name]
+
+ if (expirationDate) {
+ cloudFile.setMetaAttribute("Cache-Control", "PUBLIC, max-age=${(expirationDate.time / 1000).toInteger()}, must-revalidate")
+ cloudFile.setMetaAttribute("Expires", expirationDate)
+ }
+
+ cloudFile.bytes = localFile.bytes
+ cloudFile.save()
+ }
+ return true
+ } catch(e) {
+ log.error("Error Synchronizing With Provider ${provider}", e)
+ }
+ return false
+ }
+
+
+}
Something went wrong with that request. Please try again.