Skip to content
Permalink
Browse files

Add experimental support for WASM builds

  • Loading branch information...
Jeffail committed May 17, 2019
1 parent 36c2e7a commit 9903b3d5d8519fcf7ecbce94c336e7f054a75942
@@ -1,4 +1,4 @@
.PHONY: all deps rpm docker docker-cgo docker-push clean docs test test-race test-integration fmt lint install deploy-docs
.PHONY: all serverless deps docker docker-cgo clean docs test test-race test-integration fmt lint install deploy-docs

TAGS =

@@ -63,6 +63,8 @@ mmap_file:
retry_period: 1s
```

DEPRECATED: This buffer type is due to be removed in V3.

The mmap file buffer type uses memory mapped files to perform low-latency,
file-persisted buffering of messages.

@@ -1525,6 +1525,9 @@ The following is a list of supported drivers and their respective DSN formats:
- `mysql`: `[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]`
- `postgres`: `postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]`

Please note that the `postgres` driver enforces SSL by default, you
can override this with the parameter `sslmode=disable` if required.

## `subprocess`

``` yaml
@@ -59,18 +59,18 @@ const (

// Config is the all encompassing configuration struct for all buffer types.
type Config struct {
Type string `json:"type" yaml:"type"`
Memory single.MemoryConfig `json:"memory" yaml:"memory"`
Mmap single.MmapBufferConfig `json:"mmap_file" yaml:"mmap_file"`
None struct{} `json:"none" yaml:"none"`
Type string `json:"type" yaml:"type"`
Memory single.MemoryConfig `json:"memory" yaml:"memory"`
Mmap MmapBufferConfig `json:"mmap_file,omitempty" yaml:"mmap_file,omitempty"`
None struct{} `json:"none" yaml:"none"`
}

// NewConfig returns a configuration struct fully populated with default values.
func NewConfig() Config {
return Config{
Type: "none",
Memory: single.NewMemoryConfig(),
Mmap: single.NewMmapBufferConfig(),
Mmap: NewMmapBufferConfig(),
None: struct{}{},
}
}
@@ -18,6 +18,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package buffer

import (
@@ -28,10 +30,13 @@ import (

//------------------------------------------------------------------------------

// TODO: V3 Remove this buffer type entirely.
func init() {
Constructors[TypeMMAP] = TypeSpec{
constructor: NewMmapFile,
description: `
DEPRECATED: This buffer type is due to be removed in V3.
The mmap file buffer type uses memory mapped files to perform low-latency,
file-persisted buffering of messages.
@@ -51,10 +56,19 @@ preferably all buffers altogether.`,

//------------------------------------------------------------------------------

// MmapBufferConfig is config options for a memory-map based buffer reader.
type MmapBufferConfig single.MmapCacheConfig

// NewMmapBufferConfig creates a MmapBufferConfig oject with default values.
func NewMmapBufferConfig() MmapBufferConfig {
return MmapBufferConfig(single.NewMmapCacheConfig())
}

// NewMmapFile creates a buffer held in memory and persisted to file through
// memory map.
func NewMmapFile(config Config, log log.Modular, stats metrics.Type) (Type, error) {
b, err := single.NewMmapBuffer(config.Mmap, log, stats)
log.Warnf("The mmap_file buffer is deprecated and scheduled for removal in version 3.")
b, err := single.NewMmapBuffer(single.MmapBufferConfig(config.Mmap), log, stats)
if err != nil {
return nil, err
}
@@ -0,0 +1,35 @@
// Copyright (c) 2014 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build wasm

package buffer

//------------------------------------------------------------------------------

// MmapBufferConfig is config options for a memory-map based buffer reader.
type MmapBufferConfig interface{}

// NewMmapBufferConfig creates a MmapBufferConfig oject with default values.
func NewMmapBufferConfig() MmapBufferConfig {
return nil
}

//------------------------------------------------------------------------------
@@ -5,7 +5,7 @@
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following cacheitions:
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
@@ -18,6 +18,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package single

import (
@@ -18,6 +18,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package single

import (
@@ -21,6 +21,4 @@
// Package condition contains logical operators that, based on their
// configuration, return boolean values from messages under certain
// circumstances.
//
// TODO: V2 Move this package to ./lib
package condition
@@ -18,6 +18,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package metrics

import (
@@ -36,46 +38,6 @@ import (

//------------------------------------------------------------------------------

func init() {
Constructors[TypePrometheus] = TypeSpec{
constructor: NewPrometheus,
description: `
Host endpoints (` + "`/metrics` and `/stats`" + `) for Prometheus scraping.
Metrics paths will differ from [the list](paths.md) in that dot separators will
instead be underscores.
### Push Gateway
The field ` + "`push_url`" + ` is optional and when set will trigger a push of
metrics to a [Prometheus Push Gateway](https://prometheus.io/docs/instrumenting/pushing/)
once Benthos shuts down. It is also possible to specify a
` + "`push_interval`" + ` which results in periodic pushes.
The Push Gateway This is useful for when Benthos instances are short lived. Do
not include the "/metrics/jobs/..." path in the push URL.`,
}
}

//------------------------------------------------------------------------------

// PrometheusConfig is config for the Prometheus metrics type.
type PrometheusConfig struct {
PushURL string `json:"push_url" yaml:"push_url"`
PushInterval string `json:"push_interval" yaml:"push_interval"`
PushJobName string `json:"push_job_name" yaml:"push_job_name"`
}

// NewPrometheusConfig creates an PrometheusConfig struct with default values.
func NewPrometheusConfig() PrometheusConfig {
return PrometheusConfig{
PushURL: "",
PushInterval: "",
PushJobName: "benthos_push",
}
}

//------------------------------------------------------------------------------

// PromGauge is a representation of a single metric stat. Interactions with this
// stat are thread safe.
type PromGauge struct {
@@ -0,0 +1,63 @@
// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, sub to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package metrics

//------------------------------------------------------------------------------

func init() {
Constructors[TypePrometheus] = TypeSpec{
constructor: NewPrometheus,
description: `
Host endpoints (` + "`/metrics` and `/stats`" + `) for Prometheus scraping.
Metrics paths will differ from [the list](paths.md) in that dot separators will
instead be underscores.
### Push Gateway
The field ` + "`push_url`" + ` is optional and when set will trigger a push of
metrics to a [Prometheus Push Gateway](https://prometheus.io/docs/instrumenting/pushing/)
once Benthos shuts down. It is also possible to specify a
` + "`push_interval`" + ` which results in periodic pushes.
The Push Gateway This is useful for when Benthos instances are short lived. Do
not include the "/metrics/jobs/..." path in the push URL.`,
}
}

//------------------------------------------------------------------------------

// PrometheusConfig is config for the Prometheus metrics type.
type PrometheusConfig struct {
PushURL string `json:"push_url" yaml:"push_url"`
PushInterval string `json:"push_interval" yaml:"push_interval"`
PushJobName string `json:"push_job_name" yaml:"push_job_name"`
}

// NewPrometheusConfig creates an PrometheusConfig struct with default values.
func NewPrometheusConfig() PrometheusConfig {
return PrometheusConfig{
PushURL: "",
PushInterval: "",
PushJobName: "benthos_push",
}
}

//------------------------------------------------------------------------------
@@ -0,0 +1,34 @@
// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, sub to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build wasm

package metrics

import "errors"

//------------------------------------------------------------------------------

// NewPrometheus creates and returns a new Prometheus object.
func NewPrometheus(config Config, opts ...func(Type)) (Type, error) {
return nil, errors.New("Prometheus metrics are disabled in WASM builds")
}

//------------------------------------------------------------------------------
@@ -27,9 +27,8 @@ import (
"sync"
"time"

"github.com/Jeffail/benthos/lib/message"

"github.com/Jeffail/benthos/lib/log"
"github.com/Jeffail/benthos/lib/message"
"github.com/Jeffail/benthos/lib/message/tracing"
"github.com/Jeffail/benthos/lib/metrics"
"github.com/Jeffail/benthos/lib/types"
@@ -38,7 +37,6 @@ import (

// SQL Drivers
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
)

//------------------------------------------------------------------------------
@@ -90,7 +88,10 @@ columns value in the row.
The following is a list of supported drivers and their respective DSN formats:
- ` + "`mysql`: `[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]`" + `
- ` + "`postgres`: `postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]`" + ``,
- ` + "`postgres`: `postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]`" + `
Please note that the ` + "`postgres`" + ` driver enforces SSL by default, you
can override this with the parameter ` + "`sslmode=disable`" + ` if required.`,
}
}

@@ -0,0 +1,29 @@
// Copyright (c) 2019 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package processor

// Import extra drivers that aren't supported by WASM builds.
import (
// SQL Drivers
_ "github.com/lib/pq"
)

0 comments on commit 9903b3d

Please sign in to comment.
You can’t perform that action at this time.