Conditionally build connectors

.mergify.yml

@@ -3,7 +3,7 @@ pull_request_rules:
     conditions:
       - base=master
       - status-success=tests
-      - label!=work-in-progress
+      - "label!=work in progress"
       - "#approved-reviews-by>=1"
       - "#review-requested=0"
       - "#changes-requested-reviews-by=0"
@@ -17,7 +17,7 @@ pull_request_rules:
     conditions:
       - base=master
       - status-success=tests
-      - label!=work-in-progress
+      - "label!=work in progress"
       - author=alecmocatta # https://github.com/Mergifyio/mergify-engine/issues/451
       - "#review-requested=0"
       - "#changes-requested-reviews-by=0"
@@ -36,10 +36,10 @@ pull_request_rules:
       - "title~=^WIP: .*"
     actions:
       label:
-        add: ["work-in-progress"]
+        add: ["work in progress"]
   - name: auto remove wip label
     conditions:
       - "-title~=^WIP: .*"
     actions:
       label:
-        remove: ["work-in-progress"]
+        remove: ["work in progress"]

Cargo.toml

@@ -16,27 +16,34 @@ repository = "https://github.com/alecmocatta/amadeus"
 homepage = "https://github.com/alecmocatta/amadeus"
 documentation = "https://docs.rs/amadeus/0.1.1"
 readme = "README.md"
-autotests = true
 edition = "2018"
 
 [badges]
 azure-devops = { project = "alecmocatta/amadeus", pipeline = "tests" }
 maintenance = { status = "actively-developed" }
 
 [features]
+constellation = ["constellation-rs"]
+aws = ["amadeus-aws"]
+commoncrawl = ["amadeus-commoncrawl"]
 parquet = ["amadeus-parquet", "amadeus-derive/parquet"]
+postgres = ["amadeus-postgres", "amadeus-derive/postgres"]
+csv = ["amadeus-serde", "amadeus-derive/serde"]
+json = ["amadeus-serde", "amadeus-derive/serde"]
+
+[package.metadata.docs.rs]
+features = ["constellation", "aws", "commoncrawl", "postgres", "csv", "json"]
 
 [dependencies]
+amadeus-core = { version = "0.1.0", path = "amadeus-core" }
 amadeus-derive = { version = "0.1.0", path = "amadeus-derive" }
 amadeus-types = { version = "0.1.0", path = "amadeus-types" }
-amadeus-postgres = { version = "0.1.0", path = "amadeus-postgres" }
+amadeus-aws = { version = "0.1.0", path = "amadeus-aws", optional = true }
+amadeus-commoncrawl = { version = "0.1.0", path = "amadeus-commoncrawl", optional = true }
 amadeus-parquet = { version = "0.1.0", path = "amadeus-parquet", optional = true }
-amadeus-serde = { version = "0.1.0", path = "amadeus-serde" }
-amadeus-core = { version = "0.1.0", path = "amadeus-core" }
-amadeus-aws = { version = "0.1.0", path = "amadeus-aws" }
-amadeus-commoncrawl = { version = "0.1.0", path = "amadeus-commoncrawl" }
-constellation-rs = { version = "0.1", default-features = false }
-crossbeam-channel = "0.3"
+amadeus-postgres = { version = "0.1.0", path = "amadeus-postgres", optional = true }
+amadeus-serde = { version = "0.1.0", path = "amadeus-serde", optional = true }
+constellation-rs = { version = "0.1", default-features = false, optional = true }
 futures-preview = "=0.3.0-alpha.18"
 serde = { version = "1.0", features = ["derive"] }
 serde_closure = "0.1"
@@ -46,15 +53,14 @@ serde_traitobject = "0.1.6"
 bincode = "=1.0.1"
 
 [dev-dependencies]
-either = "1.5"
-flate2 = "1.0"
-reqwest = "0.9"
-reqwest_resume = "0.2"
+either = { version = "1.5", features = ["serde"] }
 rand = "0.7"
-rusoto_core = "0.40"
-select = "0.4"
 streaming_algorithms = "0.1"
 
+[[example]]
+name = "common_crawl"
+required-features = ["commoncrawl", "constellation"]
+
 [[test]]
 name = "into_dist_iter"
 harness = false
@@ -68,8 +74,14 @@ name = "threads"
 harness = false
 
 [[test]]
-name = "source"
+name = "cloudfront"
+harness = false
+required-features = ["aws"]
+
+[[test]]
+name = "commoncrawl"
 harness = false
+required-features = ["commoncrawl"]
 
 [[test]]
 name = "parquet"
@@ -79,12 +91,15 @@ required-features = ["parquet"]
 [[test]]
 name = "csv"
 harness = false
+required-features = ["csv"]
 
 [[test]]
 name = "json"
 harness = false
+required-features = ["json"]
 
 [[test]]
 name = "postgres"
 harness = false
+required-features = ["postgres"]
 test = false # TODO set up postgres on CI

README.md

@@ -9,8 +9,7 @@
 <p align="center">
 	<a href="https://crates.io/crates/amadeus"><img src="https://img.shields.io/crates/v/amadeus.svg?maxAge=86400" alt="Crates.io" /></a>
 	<a href="LICENSE.txt"><img src="https://img.shields.io/crates/l/amadeus.svg?maxAge=2592000" alt="Apache-2.0 licensed" /></a>
-	<a href="https://circleci.com/gh/alecmocatta/amadeus"><img src="https://circleci.com/gh/alecmocatta/amadeus/tree/master.svg?style=shield" alt="Build Status" /></a>
-	<a href="https://travis-ci.com/alecmocatta/amadeus"><img src="https://travis-ci.com/alecmocatta/amadeus.svg?branch=master" alt="Build Status" /></a>
+	<a href="https://dev.azure.com/alecmocatta/amadeus/_build/latest?branchName=master"><img src="https://dev.azure.com/alecmocatta/amadeus/_apis/build/status/tests?branchName=master" alt="Build Status" /></a>
 </p>
 
 <p align="center">

amadeus-aws/README.md

@@ -0,0 +1,3 @@
+# amadeus-aws
+
+This subcrate of the [`amadeus`](https://github.com/alecmocatta/amadeus) project includes a filesystem backend for S3 and a source for AWS Cloudfront logs.