Permalink
Browse files

proper jesque config

  • Loading branch information...
peh committed Mar 25, 2017
1 parent acc6877 commit 175e731ceffbd45f5a1ba8cb1729b01aeb577bf8
Showing with 47 additions and 69 deletions.
  1. +3 −4 build.gradle
  2. +1 −1 gradle.properties
  3. +43 −3 grails-app/conf/application.groovy
  4. +0 −61 grails-app/init/BootStrap.groovy
View
@@ -8,9 +8,10 @@ buildscript {
maven { url "https://plugins.gradle.org/m2/" }
}
dependencies {
classpath 'io.spring.gradle:dependency-management-plugin:0.6.1.RELEASE'
classpath "org.grails:grails-gradle-plugin:$grailsVersion"
classpath "com.bertramlabs.plugins:asset-pipeline-gradle:2.7.4"
// classpath 'net.errbuddy.plugins:babel-asset-pipeline:2.1.0'
classpath 'net.errbuddy.plugins:babel-asset-pipeline:2.1.0'
classpath "org.grails.plugins:hibernate4:5.0.2"
classpath 'org.grails.plugins:database-migration:2.0.0.RC4'
classpath "com.moowork.gradle:gradle-node-plugin:1.0.1"
@@ -90,7 +91,7 @@ dependencies {
compile "com.uberall.plugins:uber-doc:2.0.4"
compile 'org.grails.plugins:elasticsearch:1.0.0.2'
compile 'org.grails.plugins:grails-jesque-admin:0.1.5'
compile 'org.grails.plugins:jesque:1.2.0'
compile 'org.grails.plugins:jesque:1.2.1-UBERALL'
assets 'net.errbuddy.plugins:babel-asset-pipeline:2.1.0'
@@ -161,5 +162,3 @@ tasks.withType(JavaExec).each { task ->
}
war.archiveName "errbuddy.war"
// grails prod package -Dgradle.node.exec="/Users/philipp/.nvm/versions/node/v5.6.0/bin/node"
View
@@ -1,4 +1,4 @@
grailsVersion=3.1.4
gradleWrapperVersion=2.9
gradleWrapperVersion=3.4.1
org.gradle.daemon=true
@@ -1,4 +1,6 @@
import errbuddy.*
import grails.plugins.jesque.admin.JesqueJobStatisticsCollectingWorkerImpl
import net.greghaines.jesque.worker.LoggingWorkerListener
// Added by the Spring Security Core plugin:
grails.plugin.springsecurity.password.algorithm = 'bcrypt'
@@ -46,6 +48,24 @@ grails.gorm.default.mapping = {
"user-type" type: org.jadira.usertype.dateandtime.joda.PersistentDateTime, class: org.joda.time.DateTime
"user-type" type: org.jadira.usertype.dateandtime.joda.PersistentLocalDate, class: org.joda.time.LocalDate
}
boolean isPuttingIntoGroup = Boolean.parseBoolean(System.getProperty('errbuddy.skipPutWorker', 'true'))
def putJobs = [
ApplicationDeleteJob,
DataRetentionJob,
EntryPutJob,
EntryDeleteJob
]
def genericJobs = [
ApplicationDeploymentJob,
DeleteEmptyGroupsJob,
DeleteEntryGroupJob,
FindSimilarEntriesJob,
RefindFromCollectorJob,
ReindexJob,
]
grails {
redis {
poolConfig {
@@ -61,15 +81,35 @@ grails {
createWorkersOnStartup = true
schedulerThreadActive = true
delayedJobThreadActive = true
startPaused = true
autoFlush = true
statistics {
enabled = true
max = 100
max = 1000
}
custom {
worker.clazz = JesqueJobStatisticsCollectingWorkerImpl
listener.clazz = [net.greghaines.jesque.worker.LoggingWorkerListener]
listener.clazz = LoggingWorkerListener
}
workers {
if (isPuttingIntoGroup) {
// this should only be done by ONE worker (app wide) as we will run into concurrency issues otherwise
// it's a fairly simple and quick task so it one is enough for hundreds of put's per second
AddToGroupPool {
workers = 1
queueNames = PutIntoEntryGroupJob.queueName
jobTypes = [PutIntoEntryGroupJob.name]
}
}
PutPool {
workers = 2
queueNames = putJobs.collect { it.queueName }.unique()
jobTypes = putJobs.name
}
GenericPool {
workers = 3
queueNames = genericJobs.collect { it.queueName }.unique()
jobTypes = genericJobs.name
}
}
}
plugin {
@@ -1,53 +1,13 @@
import errbuddy.*
import grails.converters.JSON
import grails.plugins.errbuddy.ErrbuddyLogAppender
import grails.plugins.errbuddy.ErrbuddyService
import grails.plugins.jesque.JesqueService
class BootStrap {
private static final List ROLES = ['ROLE_ROOT', 'ROLE_ADMIN', 'ROLE_USER']
def grailsApplication
ApplicationService applicationService
ErrbuddyService errbuddyService
JesqueService jesqueService
private static final List WORKERS = [
[
queueName: "add_to_group",
count : 1,
jobs : [
PutIntoEntryGroupJob
]
],
[
queueName: "put",
count : 1,
jobs : [
ApplicationDeleteJob,
DataRetentionJob,
EntryPutJob,
EntryDeleteJob
]
],
[
queueName: "generic",
count : 3,
jobs : {
return [
ApplicationDeploymentJob,
DeleteEmptyGroupsJob,
DeleteEntryGroupJob,
FindSimilarEntriesJob,
MonitoringCheckJob,
MonitoringCheckCreatorJob,
RefindFromCollectorJob,
ReindexJob,
]
}
]
]
def init = { servletContext ->
JSON.registerObjectMarshaller(HasJsonBody) { HasJsonBody b ->
@@ -89,27 +49,6 @@ class BootStrap {
applicationService.create(app)
}
// if (System.getProperty('errbuddy.worker.skip') != 'true') {
new Timer().runAfter(10000) {
log.info "starting workers"
WORKERS.each { workerConfig ->
def jobs = workerConfig.jobs
if (jobs instanceof Closure) {
jobs = jobs.call()
}
if (jobs instanceof List) {
jobs = jobs.collectEntries {
return [(it.simpleName): it]
}
}
workerConfig.count.times {
jesqueService.startWorker(workerConfig.queueName as String, jobs as Map)
}
}
jesqueService.enqueue('generic', RefindFromCollectorJob)
log.info "workers started"
}
// }
}
def destroy = {

0 comments on commit 175e731

Please sign in to comment.