Skip to content

Commit

Permalink
Add settings UI and "hot reload" support to the app.
Browse files Browse the repository at this point in the history 
This is a major breaking change that moves away from having the
entire app configuration in external TOML files to settings being
in the database with a UI to update them dynamically.

The app loads all config into memory (app settings, SMTP conf)
on boot. "Hot" replacing them is complex and it's a fair tradeoff
to instead just restart the application as it is practically
instant.

A new `settings` table stores arbitrary string keys with a JSONB
value field which happens to support arbitrary types. After every
settings update, the app gracefully releases all resources
(HTTP server, DB pool, SMTP pool etc.) and restarts itself,
occupying the same PID. If there are any running campaigns, the
auto-restart doesn't happen and the user is prompted to invoke
it manually with a one-click button once all running campaigns
have been paused.
  • Loading branch information
@knadh
knadh committed Jul 20, 2020
1 parent d294c95 commit 942eb7c
Show file tree
Hide file tree
Showing 27 changed files with 1,143 additions and 372 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Expand Up @@ -20,7 +20,7 @@ deps:
# Build steps.
.PHONY: build
build:
go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'"
go build -o ${BIN} -ldflags="-s -w -X 'main.buildString=${BUILDSTR}'"

.PHONY: build-frontend
build-frontend:
Expand Down
22 changes: 20 additions & 2 deletions admin.go
View file
Expand Up @@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"
"net/http"
"syscall"
"time"

"github.com/jmoiron/sqlx/types"
"github.com/labstack/echo"
Expand All @@ -14,7 +16,8 @@ type configScript struct {
RootURL string `json:"rootURL"`
FromEmail string `json:"fromEmail"`
Messengers []string `json:"messengers"`
MediaProvider string `json:"media_provider"`
MediaProvider string `json:"mediaProvider"`
NeedsRestart bool `json:"needsRestart"`
}

// handleGetConfigScript returns general configuration as a Javascript
Expand All @@ -28,11 +31,16 @@ func handleGetConfigScript(c echo.Context) error {
Messengers: app.manager.GetMessengerNames(),
MediaProvider: app.constants.MediaProvider,
}
)

app.Lock()
out.NeedsRestart = app.needsRestart
app.Unlock()

var (
b = bytes.Buffer{}
j = json.NewEncoder(&b)
)

b.Write([]byte(`var CONFIG = `))
_ = j.Encode(out)
return c.Blob(http.StatusOK, "application/javascript", b.Bytes())
Expand Down Expand Up @@ -67,3 +75,13 @@ func handleGetDashboardCounts(c echo.Context) error {

return c.JSON(http.StatusOK, okResp{out})
}

// handleReloadApp restarts the app.
func handleReloadApp(c echo.Context) error {
app := c.Get("app").(*App)
go func() {
<-time.After(time.Millisecond * 500)
app.sigChan <- syscall.SIGHUP
}()
return c.JSON(http.StatusOK, okResp{true})
}
203 changes: 8 additions & 195 deletions config.toml.sample
View file
@@ -1,199 +1,12 @@
[app]
# Interface and port where the app will run its webserver.
address = "0.0.0.0:9000"

# Public root URL of the listmonk installation that'll be used
# in the messages for linking to images, unsubscribe page etc.
root = "https://listmonk.mysite.com"

# (Optional) full URL to the static logo to be displayed on
# user facing view such as the unsubscription page.
# eg: https://mysite.com/images/logo.svg
logo_url = "https://listmonk.mysite.com/public/static/logo.png"

# (Optional) full URL to the static favicon to be displayed on
# user facing view such as the unsubscription page.
# eg: https://mysite.com/images/favicon.png
favicon_url = "https://listmonk.mysite.com/public/static/favicon.png"

# The default 'from' e-mail for outgoing e-mail campaigns.
from_email = "listmonk <from@mail.com>"

# List of e-mail addresses to which admin notifications such as
# import updates, campaign completion, failure etc. should be sent.
# To disable notifications, set an empty list, eg: notify_emails = []
notify_emails = ["admin1@mysite.com", "admin2@mysite.com"]

# Maximum concurrent workers that will attempt to send messages
# simultaneously. This should ideally depend on the number of CPUs
# available, and should be based on the maximum number of messages
# a target SMTP server will accept.
concurrency = 5

# Maximum number of messages to be sent out per second per worker.
# If concurrency = 10 and message_rate = 10, then up to 10x10=100 messages
# may be pushed out every second. This, along with concurrency, should be
# tweaked to keep the net messages going out per second under the target
# SMTP's rate limits, if any.
message_rate = 5

# The number of errors (eg: SMTP timeouts while e-mailing) a running
# campaign should tolerate before it is paused for manual
# investigation or intervention. Set to 0 to never pause.
max_send_errors = 1000

# The number of subscribers to pull from the databse in a single iteration.
# Each iteration pulls subscribers from the database, sends messages to them,
# and then moves on to the next iteration to pull the next batch.
# This should ideally be higher than the maximum achievable throughput (concurrency * message_rate)
batch_size = 1000

[privacy]
# Allow subscribers to unsubscribe from all mailing lists and mark themselves
# as blacklisted?
allow_blacklist = false

# Allow subscribers to export data recorded on them?
allow_export = false

# Items to include in the data export.
# profile Subscriber's profile including custom attributes
# subscriptions Subscriber's subscription lists (private list names are masked)
# campaign_views Campaigns the subscriber has viewed and the view counts
# link_clicks Links that the subscriber has clicked and the click counts
exportable = ["profile", "subscriptions", "campaign_views", "link_clicks"]

# Allow subscribers to delete themselves from the database?
# This deletes the subscriber and all their subscriptions.
# Their association to campaign views and link clicks are also
# removed while views and click counts remain (with no subscriber
# associated to them) so that stats and analytics aren't affected.
allow_wipe = false

# Interface and port where the app will run its webserver.
address = "0.0.0.0:9000"

# Database.
[db]
host = "db"
port = 5432
user = "listmonk"
password = "listmonk"
database = "listmonk"
ssl_mode = "disable"

# Maximum active and idle connections to pool.
max_open = 50
max_idle = 10

# SMTP servers.
[smtp]
[smtp.my0]
enabled = true
host = "my.smtp.server"
port = 25

# "cram", "plain", or "login". Empty string for no auth.
auth_protocol = "cram"
username = "xxxxx"
password = ""

# Format to send e-mails in: html|plain|both.
email_format = "both"

# Optional. Some SMTP servers require a FQDN in the hostname.
# By default, HELLOs go with "localhost". Set this if a custom
# hostname should be used.
hello_hostname = ""

# Maximum concurrent connections to the SMTP server.
max_conns = 10

# Time to wait for new activity on a connection before closing
# it and removing it from the pool.
idle_timeout = "15s"

# Message send / wait timeout.
wait_timeout = "5s"

# The number of times a message should be retried if sending fails.
max_msg_retries = 2

# Enable STARTTLS.
tls_enabled = true
tls_skip_verify = false

# One or more optional custom headers to be attached to all e-mails
# sent from this SMTP server. Uncomment the line to enable.
# email_headers = { "X-Sender" = "listmonk", "X-Custom-Header" = "listmonk" }

[smtp.postal]
enabled = false
host = "my.smtp.server2"
port = 25

# cram or plain.
auth_protocol = "plain"
username = "xxxxx"
password = ""

# Format to send e-mails in: html|plain|both.
email_format = "both"

# Optional. Some SMTP servers require a FQDN in the hostname.
# By default, HELLOs go with "localhost". Set this if a custom
# hostname should be used.
hello_hostname = ""

# Maximum concurrent connections to the SMTP server.
max_conns = 10

# Time to wait for new activity on a connection before closing
# it and removing it from the pool.
idle_timeout = "15s"

# Message send / wait timeout.
wait_timeout = "5s"

# The number of times a message should be retried if sending fails.
max_msg_retries = 2

# Enable STARTTLS.
tls_enabled = true
tls_skip_verify = false

[upload]
# File storage backend. "filesystem" or "s3".
provider = "filesystem"

[upload.s3]
# (Optional). AWS Access Key and Secret Key for the user to access the bucket.
# Leaving it empty would default to use instance IAM role.
aws_access_key_id = ""
aws_secret_access_key = ""

# AWS Region where S3 bucket is hosted.
aws_default_region = "ap-south-1"

# Bucket name.
bucket = ""

# Path where the files will be stored inside bucket. Default is "/".
bucket_path = "/"

# Optional full URL to the bucket. eg: https://files.mycustom.com
bucket_url = ""

# "private" or "public".
bucket_type = "public"

# (Optional) Specify TTL (in seconds) for the generated presigned URL.
# Expiry value is used only if the bucket is private.
expiry = 86400

[upload.filesystem]
# Path to the uploads directory where media will be uploaded.
upload_path="./uploads"

# Upload URI that's visible to the outside world.
# The media uploaded to upload_path will be made available publicly
# under this URI, for instance, list.yoursite.com/uploads.
upload_uri = "/uploads"
host = "db"
port = 5432
user = "listmonk"
password = "listmonk"
database = "listmonk"
ssl_mode = "disable"
47 changes: 39 additions & 8 deletions frontend/src/App.vue
View file
Expand Up @@ -63,9 +63,9 @@
icon="file-image-outline" label="Templates"></b-menu-item>
</b-menu-item><!-- campaigns -->

<!-- <b-menu-item :to="{name: 'settings'}" tag="router-link"
<b-menu-item :to="{name: 'settings'}" tag="router-link"
:active="activeItem.settings"
icon="cog-outline" label="Settings"></b-menu-item> -->
icon="cog-outline" label="Settings"></b-menu-item>
</b-menu-list>
</b-menu>
</div>
Expand All @@ -75,15 +75,27 @@

<!-- body //-->
<div class="main">
<div class="global-notices" v-if="serverConfig.needsRestart">
<div v-if="serverConfig.needsRestart" class="notification is-danger">
Settings have changed. Pause all running campaigns and restart the app
&mdash;
<b-button class="is-primary" size="is-small"
@click="$utils.confirm(
'Ensure running campaigns are paused. Restart?', reloadApp)">
Restart
</b-button>
</div>
</div>

<router-view :key="$route.fullPath" />
</div>

<b-loading v-if="!isLoaded" active>
<div class="has-text-centered">
<h1 class="title">Oops</h1>
<p>
Can't connect to the listmonk backend.<br />
Make sure it is running and refresh this page.
Can't connect to the backend.<br />
Make sure the server is running and refresh this page.
</p>
</div>
</b-loading>
Expand All @@ -92,6 +104,7 @@

<script>
import Vue from 'vue';
import { mapState } from 'vuex';
export default Vue.extend({
name: 'App',
Expand All @@ -115,17 +128,35 @@ export default Vue.extend({
},
},
mounted() {
// Lists is required across different views. On app load, fetch the lists
// and have them in the store.
this.$api.getLists();
methods: {
reloadApp() {
this.$api.reloadApp().then(() => {
this.$utils.toast('Reloading app ...');
// Poll until there's a 200 response, waiting for the app
// to restart and come back up.
const pollId = setInterval(() => {
clearInterval(pollId);
this.$utils.toast('Reload complete');
document.location.reload();
}, 500);
});
},
},
computed: {
...mapState(['serverConfig']),
version() {
return process.env.VUE_APP_VERSION;
},
},
mounted() {
// Lists is required across different views. On app load, fetch the lists
// and have them in the store.
this.$api.getLists();
},
});
</script>

Expand Down

0 comments on commit 942eb7c

Please sign in to comment.