diff --git a/CHANGELOG.md b/CHANGELOG.md index 1432bfe99d..b269ea53d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Cloud** Support multipart uploads for builds - **Infra** Support configuring multiple S3 providers - **Infra** Support multipart uploads - **Infra** Replace Promtail-based log shipping with native Loki Docker driver -- **Infra** Add local Traefik Cloudflare proxy daemon for connecting to Cloudflare Access services +- **Infra** Local Traefik Cloudflare proxy daemon for connecting to Cloudflare Access services +- **Infra** Upload service builds to default S3 provider instead of hardcoded bucket - **Bolt** Support for connecting to Redis databases with `bolt redis sh` -- **Bolt** Add confirmation before running any command in the production namespace +- **Bolt** Confirmation before running any command in the production namespace +- **Bolt** `--start-at` flag for all infra commands ### Changed @@ -31,6 +34,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Infra** Update Redis Exporter to 1.52.0 - **Infra** Update Redis to 7.0.12 - **Infra** Update Treafik to 2.10.4 +- **Bolt** PostHog events are now captured in a background task +- **Bolt** Auto-install rsync on Salt Master +- **Bolt** Recursively add dependencies from overridden services when using additional roots +- **KV** Significantly rate limit of all endpoints ### Security @@ -40,4 +47,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Portal** Skip captcha if no Turnstile key provided - **Infra** Resolve [RUSTSEC-2023-0044](https://rustsec.org/advisories/RUSTSEC-2023-0044) +- **Infra** Missing dpenedency on mounting volumn before setting permissions of /var/* for Cockroach, ClickHouse, Prometheus, and Traffic Server +- **Chrip** Empty message parameters now have placeholder so NATS doesn't throw an error +- **Chrip** Messages with no parameters no longer have a trailing dot - **Bolt** Correctly resolve project root when building services natively +- **Bolt** Correctly determine executable path for `ExecServiceDriver::UploadedBinaryArtifact` with different Cargo names + diff --git a/docs/getting_started/INTERNAL_DASHBOARDS.md b/docs/getting_started/INTERNAL_DASHBOARDS.md index 5ff86616ab..bb907fa1ed 100644 --- a/docs/getting_started/INTERNAL_DASHBOARDS.md +++ b/docs/getting_started/INTERNAL_DASHBOARDS.md @@ -6,14 +6,14 @@ Exposed tunnels & applications are configured [here](/lib/bolt/core/src/dep/terr Replace `MAIN_DOMAIN` with the value of `dns.domain.main`. -- [Consul](https://consul.MAIN_DOMAIN)) -- [Nomad](https://nomad.MAIN_DOMAIN)) -- [Cockroach](https://cockroach-http.MAIN_DOMAIN)) -- [ClickHouse](https://clickhouse-http.MAIN_DOMAIN)) -- [Prometheus (svc)](https://prometheus-svc.MAIN_DOMAIN)) -- [Prometheus (job)](https://prometheus-job.MAIN_DOMAIN)) -- [Minio](https://minio-console.MAIN_DOMAIN)) -- [Traefik (proxied)](https://ing-px.MAIN_DOMAIN)) -- [Traefik (job)](https://ing-job.MAIN_DOMAIN)) +- [Consul](https://consul.MAIN_DOMAIN) +- [Nomad](https://nomad.MAIN_DOMAIN) +- [Cockroach](https://cockroach-http.MAIN_DOMAIN) +- [ClickHouse](https://clickhouse-http.MAIN_DOMAIN) +- [Prometheus (svc)](https://prometheus-svc.MAIN_DOMAIN) +- [Prometheus (job)](https://prometheus-job.MAIN_DOMAIN) +- [Minio](https://minio-console.MAIN_DOMAIN) +- [Traefik (proxied)](https://ing-px.MAIN_DOMAIN) +- [Traefik (job)](https://ing-job.MAIN_DOMAIN) - This does not support regional dashboards at the moment (SVC-2584) - Will choose a random region until fixed diff --git a/docs/infrastructure/nats/TROUBLESHOOTING.md b/docs/infrastructure/nats/TROUBLESHOOTING.md new file mode 100644 index 0000000000..27f9acd3bc --- /dev/null +++ b/docs/infrastructure/nats/TROUBLESHOOTING.md @@ -0,0 +1,10 @@ +# Troubleshooting + +## Checking the health of the cluster manually... + +1. `bolt ssh pool nats` +2. `nix-shell -p natscli` +3. `nats --server=10.0.44.2:4222 --user admin --password password context save default` +4. `nats context select default` +5. `nats server report connections` + diff --git a/docs/infrastructure/saltstack/TROUBLESHOOTING.md b/docs/infrastructure/saltstack/TROUBLESHOOTING.md index 331675e0d9..0f624cdcec 100644 --- a/docs/infrastructure/saltstack/TROUBLESHOOTING.md +++ b/docs/infrastructure/saltstack/TROUBLESHOOTING.md @@ -31,3 +31,15 @@ Try a few things to figure this out: - Run `pstree -p my-pid` on the `salt-minion` process to see what subcommand is being ran - Read the `salt-minion` logs with `journalctl -u salt-minion` - Try applying specific SLS files with `salt apply 'my-minion' --sls my_file` + +## Error when bootstrapping Minion: `RSA key format is not supported` + +```bash +# Uninstall Salt +bolt ssh name staging-lnd-atl-crdb-05-2 'systemctl stop salt-minion; apt remove -y salt-cloud salt-common salt-minion; rm -rf /etc/salt /opt/saltstack /var/log/salt /var/cache/salt /run/salt /usr/bin/salt-*; echo Done' + +# Re-run install_salt_minion +(cd infra/tf/pools && terraform state rm 'module.install_salt_minion["staging-lnd-atl-crdb-05-2"]') +bolt tf apply pools +``` + diff --git a/fern/api/definition/cloud/games/builds.yml b/fern/api/definition/cloud/games/builds.yml index 0fe300e243..b6e3caac62 100644 --- a/fern/api/definition/cloud/games/builds.yml +++ b/fern/api/definition/cloud/games/builds.yml @@ -44,6 +44,8 @@ types: docs: A tag given to the game build. type: string image_file: uploadCommons.PrepareFile + multipart_upload: + type: optional CreateGameBuildResponse: properties: @@ -51,8 +53,6 @@ types: type: uuid upload_id: type: uuid - image_presigned_request: - docs: >- - **Deprecated: use image_presigned_requests instead** - type: uploadCommons.PresignedRequest - image_presigned_requests: list + image_presigned_request: optional + image_presigned_requests: optional> + diff --git a/gen/openapi/external/spec/openapi.yml b/gen/openapi/external/spec/openapi.yml index c24c4c0d10..80f3da3f2f 100644 --- a/gen/openapi/external/spec/openapi.yml +++ b/gen/openapi/external/spec/openapi.yml @@ -9882,6 +9882,8 @@ components: description: A tag given to the game build. image_file: $ref: '#/components/schemas/UploadPrepareFile' + multipart_upload: + type: boolean required: - display_name - image_tag @@ -9897,7 +9899,6 @@ components: format: uuid image_presigned_request: $ref: '#/components/schemas/UploadPresignedRequest' - description: '**Deprecated: use image_presigned_requests instead**' image_presigned_requests: type: array items: @@ -9905,8 +9906,6 @@ components: required: - build_id - upload_id - - image_presigned_request - - image_presigned_requests CloudGamesListGameCdnSitesResponse: type: object properties: diff --git a/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildRequest.md b/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildRequest.md index 52a1167c8b..0e0024eb05 100644 --- a/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildRequest.md +++ b/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildRequest.md @@ -7,6 +7,7 @@ Name | Type | Description | Notes **display_name** | **String** | Represent a resource's readable display name. | **image_file** | [**crate::models::UploadPrepareFile**](UploadPrepareFile.md) | | **image_tag** | **String** | A tag given to the game build. | +**multipart_upload** | Option<**bool**> | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildResponse.md b/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildResponse.md index da36496605..fb3c4a586a 100644 --- a/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildResponse.md +++ b/gen/openapi/internal/rust/docs/CloudGamesCreateGameBuildResponse.md @@ -5,8 +5,8 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **build_id** | [**uuid::Uuid**](uuid::Uuid.md) | | -**image_presigned_request** | [**crate::models::UploadPresignedRequest**](UploadPresignedRequest.md) | | -**image_presigned_requests** | [**Vec**](UploadPresignedRequest.md) | | +**image_presigned_request** | Option<[**crate::models::UploadPresignedRequest**](UploadPresignedRequest.md)> | | [optional] +**image_presigned_requests** | Option<[**Vec**](UploadPresignedRequest.md)> | | [optional] **upload_id** | [**uuid::Uuid**](uuid::Uuid.md) | | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_request.rs b/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_request.rs index 64139e9a8f..f15f843689 100644 --- a/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_request.rs +++ b/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_request.rs @@ -21,6 +21,8 @@ pub struct CloudGamesCreateGameBuildRequest { /// A tag given to the game build. #[serde(rename = "image_tag")] pub image_tag: String, + #[serde(rename = "multipart_upload", skip_serializing_if = "Option::is_none")] + pub multipart_upload: Option, } impl CloudGamesCreateGameBuildRequest { @@ -29,6 +31,7 @@ impl CloudGamesCreateGameBuildRequest { display_name, image_file: Box::new(image_file), image_tag, + multipart_upload: None, } } } diff --git a/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_response.rs b/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_response.rs index 90a2191d66..bf7ed65cca 100644 --- a/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_response.rs +++ b/gen/openapi/internal/rust/src/models/cloud_games_create_game_build_response.rs @@ -15,20 +15,20 @@ pub struct CloudGamesCreateGameBuildResponse { #[serde(rename = "build_id")] pub build_id: uuid::Uuid, - #[serde(rename = "image_presigned_request")] - pub image_presigned_request: Box, - #[serde(rename = "image_presigned_requests")] - pub image_presigned_requests: Vec, + #[serde(rename = "image_presigned_request", skip_serializing_if = "Option::is_none")] + pub image_presigned_request: Option>, + #[serde(rename = "image_presigned_requests", skip_serializing_if = "Option::is_none")] + pub image_presigned_requests: Option>, #[serde(rename = "upload_id")] pub upload_id: uuid::Uuid, } impl CloudGamesCreateGameBuildResponse { - pub fn new(build_id: uuid::Uuid, image_presigned_request: crate::models::UploadPresignedRequest, image_presigned_requests: Vec, upload_id: uuid::Uuid) -> CloudGamesCreateGameBuildResponse { + pub fn new(build_id: uuid::Uuid, upload_id: uuid::Uuid) -> CloudGamesCreateGameBuildResponse { CloudGamesCreateGameBuildResponse { build_id, - image_presigned_request: Box::new(image_presigned_request), - image_presigned_requests, + image_presigned_request: None, + image_presigned_requests: None, upload_id, } } diff --git a/gen/openapi/internal/spec/openapi.yml b/gen/openapi/internal/spec/openapi.yml index 29beeb0a78..aa698c9cfb 100644 --- a/gen/openapi/internal/spec/openapi.yml +++ b/gen/openapi/internal/spec/openapi.yml @@ -10927,6 +10927,8 @@ components: description: A tag given to the game build. image_file: $ref: '#/components/schemas/UploadPrepareFile' + multipart_upload: + type: boolean required: - display_name - image_tag @@ -10942,7 +10944,6 @@ components: format: uuid image_presigned_request: $ref: '#/components/schemas/UploadPresignedRequest' - description: '**Deprecated: use image_presigned_requests instead**' image_presigned_requests: type: array items: @@ -10950,8 +10951,6 @@ components: required: - build_id - upload_id - - image_presigned_request - - image_presigned_requests CloudGamesListGameCdnSitesResponse: type: object properties: diff --git a/gen/openapi/internal/spec_compat/openapi.yml b/gen/openapi/internal/spec_compat/openapi.yml index 9684cdaffa..e53999a318 100644 --- a/gen/openapi/internal/spec_compat/openapi.yml +++ b/gen/openapi/internal/spec_compat/openapi.yml @@ -756,6 +756,8 @@ components: image_tag: description: A tag given to the game build. type: string + multipart_upload: + type: boolean required: - display_name - image_tag @@ -768,7 +770,6 @@ components: type: string image_presigned_request: $ref: '#/components/schemas/UploadPresignedRequest' - description: '**Deprecated: use image_presigned_requests instead**' image_presigned_requests: items: $ref: '#/components/schemas/UploadPresignedRequest' @@ -779,8 +780,6 @@ components: required: - build_id - upload_id - - image_presigned_request - - image_presigned_requests type: object CloudGamesCreateGameCdnSiteRequest: properties: diff --git a/infra/salt/pillar/ats/init.sls b/infra/salt/pillar/ats/init.sls new file mode 100644 index 0000000000..9ff3ca83b1 --- /dev/null +++ b/infra/salt/pillar/ats/init.sls @@ -0,0 +1,3 @@ +{% import_json "/srv/salt-context/rivet/secrets.json" as rivet_secrets %} + +ats: {{ rivet_secrets['ats'] }} diff --git a/infra/salt/pillar/top.sls b/infra/salt/pillar/top.sls index 9256acf146..30d85ac326 100644 --- a/infra/salt/pillar/top.sls +++ b/infra/salt/pillar/top.sls @@ -11,6 +11,7 @@ base: - nomad 'G@roles:traffic-server': - s3 + - ats 'G@roles:traefik': - tls - api-route @@ -18,6 +19,6 @@ base: - clickhouse 'G@roles:minio': - minio - 'G@roles:traefik-cloudflare-proxy': + 'G@roles:cloudflare-proxy': - cloudflare diff --git a/infra/salt/salt/clickhouse/files/clickhouse-server.service b/infra/salt/salt/clickhouse/files/clickhouse-server.service index 62aa5d00aa..4999f0ddb3 100644 --- a/infra/salt/salt/clickhouse/files/clickhouse-server.service +++ b/infra/salt/salt/clickhouse/files/clickhouse-server.service @@ -21,7 +21,7 @@ ExecStart=/var/rivet-nix/result/clickhouse/bin/clickhouse server --config=/etc/c LimitCORE=infinity LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/clickhouse/init.sls b/infra/salt/salt/clickhouse/init.sls index c23ffdcec3..ab3af39001 100644 --- a/infra/salt/salt/clickhouse/init.sls +++ b/infra/salt/salt/clickhouse/init.sls @@ -1,3 +1,23 @@ +{% if grains['volumes']['ch']['mount'] %} +{% set device = '/dev/disk/by-id/scsi-0Linode_Volume_' ~ grains['rivet']['name'] ~ '-ch' %} + +disk_create_clickhouse: + blockdev.formatted: + - name: {{ device }} + - fs_type: ext4 + +disk_mount_clickhouse: + file.directory: + - name: /var/lib/clickhouse + - makedirs: True + mount.mounted: + - name: /var/lib/clickhouse + - device: {{ device }} + - fstype: ext4 + - require: + - blockdev: disk_create_clickhouse +{% endif %} + create_clickhouse_user: user.present: - name: clickhouse @@ -26,24 +46,16 @@ mkdir_clickhouse: - mode: 700 - require: - user: create_clickhouse_user + {%- if grains['volumes']['ch']['mount'] %} + - mount: disk_mount_clickhouse + {%- endif %} -{% if grains['volumes']['ch']['mount'] %} -{% set device = '/dev/disk/by-id/scsi-0Linode_Volume_' ~ grains['rivet']['name'] ~ '-ch' %} - -disk_create_clickhouse: - blockdev.formatted: - - name: {{ device }} - - fs_type: ext4 - -disk_mount_clickhouse: - mount.mounted: - - name: /var/lib/clickhouse - - device: {{ device }} - - fstype: ext4 - - require: - - blockdev: disk_create_clickhouse - - file: mkdir_clickhouse -{% endif %} +# Remove old config directories with residual files +remove_etc_clickhouse_server_dirs: + file.absent: + - names: + - /etc/clickhouse-server/config.d + - /etc/clickhouse-server/users.d push_etc_clickhouse_server: file.managed: diff --git a/infra/salt/salt/traefik_cloudflare_proxy/files/traefik_cloudflare_proxy.service b/infra/salt/salt/cloudflare_proxy/files/cloudflare-proxy.service similarity index 73% rename from infra/salt/salt/traefik_cloudflare_proxy/files/traefik_cloudflare_proxy.service rename to infra/salt/salt/cloudflare_proxy/files/cloudflare-proxy.service index 4109a13f9a..0947b52340 100644 --- a/infra/salt/salt/traefik_cloudflare_proxy/files/traefik_cloudflare_proxy.service +++ b/infra/salt/salt/cloudflare_proxy/files/cloudflare-proxy.service @@ -4,9 +4,9 @@ After=network-online.target Wants=network-online.target systemd-networkd-wait-online.service [Service] -User=traefik_cloudflare_proxy -Group=traefik_cloudflare_proxy -ExecStart=/usr/bin/traefik --configFile=/etc/traefik_cloudflare_proxy/traefik.toml +User=cloudflare_proxy +Group=cloudflare_proxy +ExecStart=/usr/bin/traefik --configFile=/etc/cloudflare_proxy/traefik.toml PrivateTmp=true PrivateDevices=false ProtectHome=true diff --git a/infra/salt/salt/cloudflare_proxy/files/consul/cloudflare-proxy.hcl b/infra/salt/salt/cloudflare_proxy/files/consul/cloudflare-proxy.hcl new file mode 100644 index 0000000000..b562dd36ce --- /dev/null +++ b/infra/salt/salt/cloudflare_proxy/files/consul/cloudflare-proxy.hcl @@ -0,0 +1,15 @@ +services { + name = "cloudflare-proxy" + tags = ["traefik"] + + port = 9060 + checks = [ + { + name = "Reachable on 9060" + tcp = "127.0.0.1:9060" + interval = "10s" + timeout = "1s" + } + ] +} + diff --git a/infra/salt/salt/traefik_cloudflare_proxy/files/dynamic/common.toml.j2 b/infra/salt/salt/cloudflare_proxy/files/dynamic/common.toml.j2 similarity index 100% rename from infra/salt/salt/traefik_cloudflare_proxy/files/dynamic/common.toml.j2 rename to infra/salt/salt/cloudflare_proxy/files/dynamic/common.toml.j2 diff --git a/infra/salt/salt/traefik_cloudflare_proxy/files/traefik.toml.j2 b/infra/salt/salt/cloudflare_proxy/files/traefik.toml.j2 similarity index 63% rename from infra/salt/salt/traefik_cloudflare_proxy/files/traefik.toml.j2 rename to infra/salt/salt/cloudflare_proxy/files/traefik.toml.j2 index dfb3c3dba4..c6194f5cc7 100644 --- a/infra/salt/salt/traefik_cloudflare_proxy/files/traefik.toml.j2 +++ b/infra/salt/salt/cloudflare_proxy/files/traefik.toml.j2 @@ -4,4 +4,4 @@ [providers] [providers.file] - directory = "/etc/traefik_cloudflare_proxy/dynamic" + directory = "/etc/cloudflare_proxy/dynamic" diff --git a/infra/salt/salt/cloudflare_proxy/init.sls b/infra/salt/salt/cloudflare_proxy/init.sls new file mode 100644 index 0000000000..ffcf9456e2 --- /dev/null +++ b/infra/salt/salt/cloudflare_proxy/init.sls @@ -0,0 +1,72 @@ +create_cloudflare_proxy_user: + user.present: + - name: cloudflare_proxy + - shell: /bin/false + - system: True + - usergroup: True + +create_etc_cloudflare_proxy: + file.directory: + - names: + - /etc/cloudflare_proxy/: {} + - /etc/cloudflare_proxy/dynamic/: {} + - user: cloudflare_proxy + - group: cloudflare_proxy + - mode: 550 + - require: + - user: create_cloudflare_proxy_user + +push_etc_cloudflare_proxy: + file.managed: + - names: + # Static config + - /etc/cloudflare_proxy/traefik.toml: + - source: salt://cloudflare_proxy/files/traefik.toml.j2 + + # Dynamic configs + - /etc/cloudflare_proxy/dynamic/common.toml: + - source: salt://cloudflare_proxy/files/dynamic/common.toml.j2 + + - user: cloudflare_proxy + - group: cloudflare_proxy + - template: jinja + - context: + loki_url: "{{ pillar['rivet']['logging']['loki']['endpoint'] }}" + access_client_id: "{{ pillar['cloudflare']['access']['proxy']['client_id'] }}" + access_client_secret: "{{ pillar['cloudflare']['access']['proxy']['client_secret'] }}" + - mode: 440 + - require: + - file: create_etc_cloudflare_proxy + +push_cloudflare_proxy_service: + file.managed: + - name: /etc/systemd/system/cloudflare-proxy.service + - source: salt://cloudflare_proxy/files/cloudflare-proxy.service + +# Manually restart the Traefik service yourself in order to prevent terminating +# connections needlessly +start_cloudflare_proxy_service: + service.running: + - name: cloudflare-proxy + - enable: True + - require: + - file: push_cloudflare_proxy_service + - file: push_etc_cloudflare_proxy + +{%- if 'consul-client' in grains['roles'] %} +push_etc_consul_cloudflare_proxy_hcl: + file.managed: + - name: /etc/consul.d/cloudflare-proxy.hcl + - source: salt://cloudflare_proxy/files/consul/cloudflare-proxy.hcl + - template: jinja + - require: + - file: create_etc_consul + cmd.run: + - name: consul reload + - require: + - service: start_consul_service + - file: push_etc_consul_cloudflare_proxy_hcl + - onchanges: + - file: push_etc_consul_cloudflare_proxy_hcl +{%- endif %} + diff --git a/infra/salt/salt/cloudflared/files/cloudflared.service b/infra/salt/salt/cloudflared/files/cloudflared.service index 686d901868..c9bd5f7e1a 100644 --- a/infra/salt/salt/cloudflared/files/cloudflared.service +++ b/infra/salt/salt/cloudflared/files/cloudflared.service @@ -8,7 +8,7 @@ Type=notify ExecStart=/var/rivet-nix/result/cloudflared/bin/cloudflared --no-autoupdate --config /etc/cloudflared/config.yml tunnel run Restart=on-failure RestartSec=5s -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/cockroach/files/cockroach.service.j2 b/infra/salt/salt/cockroach/files/cockroach.service.j2 index 06886385b5..b77136a8bd 100644 --- a/infra/salt/salt/cockroach/files/cockroach.service.j2 +++ b/infra/salt/salt/cockroach/files/cockroach.service.j2 @@ -28,7 +28,7 @@ ExecStart=/usr/local/bin/cockroach start \ TimeoutStopSec=60 Restart=always RestartSec=10 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/cockroach/init.sls b/infra/salt/salt/cockroach/init.sls index f4282b3450..2454d22d32 100644 --- a/infra/salt/salt/cockroach/init.sls +++ b/infra/salt/salt/cockroach/init.sls @@ -13,13 +13,14 @@ disk_create_cockroach: - fs_type: ext4 disk_mount_cockroach: + file.directory: + - name: /var/lib/cockroach mount.mounted: - name: /var/lib/cockroach - device: {{ device }} - fstype: ext4 - require: - blockdev: disk_create_cockroach - - file: mkdir_crdb {% endif %} @@ -67,6 +68,9 @@ mkdir_crdb: - /usr/local/lib/cockroach: [] - require: - user: create_crdb_user + {%- if grains['volumes']['crdb']['mount'] %} + - mount: disk_mount_cockroach + {%- endif %} install_crdb: diff --git a/infra/salt/salt/consul/files/consul.service b/infra/salt/salt/consul/files/consul.service index 9ac93b15d9..cc929ebc64 100644 --- a/infra/salt/salt/consul/files/consul.service +++ b/infra/salt/salt/consul/files/consul.service @@ -24,7 +24,7 @@ KillSignal=SIGTERM LimitNOFILE=65536 Restart=always RestartSec=2 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/traefik/files/consul/traefik.hcl.j2 b/infra/salt/salt/ingress_proxy/files/consul/ingress-proxy.hcl.j2 similarity index 87% rename from infra/salt/salt/traefik/files/consul/traefik.hcl.j2 rename to infra/salt/salt/ingress_proxy/files/consul/ingress-proxy.hcl.j2 index aa61b4a163..abfc7d2f6f 100644 --- a/infra/salt/salt/traefik/files/consul/traefik.hcl.j2 +++ b/infra/salt/salt/ingress_proxy/files/consul/ingress-proxy.hcl.j2 @@ -1,5 +1,5 @@ services { - id = "traefik-lb-80" + id = "ingress-proxy-lb-80" name = "{{service_name}}" tags = ["traefik", "lb-80"] @@ -15,7 +15,7 @@ services { } services { - id = "traefik-lb-443" + id = "ingress-proxy-lb-443" name = "{{service_name}}" tags = ["traefik", "lb-443"] @@ -31,7 +31,7 @@ services { } services { - id = "traefik-client" + id = "ingress-proxy-client" name = "{{service_name}}" tags = ["traefik", "api"] diff --git a/infra/salt/salt/traefik/files/dynamic/common.toml.j2 b/infra/salt/salt/ingress_proxy/files/dynamic/common.toml.j2 similarity index 100% rename from infra/salt/salt/traefik/files/dynamic/common.toml.j2 rename to infra/salt/salt/ingress_proxy/files/dynamic/common.toml.j2 diff --git a/infra/salt/salt/traefik/files/dynamic/tls.toml.j2 b/infra/salt/salt/ingress_proxy/files/dynamic/tls.toml.j2 similarity index 53% rename from infra/salt/salt/traefik/files/dynamic/tls.toml.j2 rename to infra/salt/salt/ingress_proxy/files/dynamic/tls.toml.j2 index 87018329cb..b5a536b690 100644 --- a/infra/salt/salt/traefik/files/dynamic/tls.toml.j2 +++ b/infra/salt/salt/ingress_proxy/files/dynamic/tls.toml.j2 @@ -1,12 +1,12 @@ {%- for cert_id in traefik.tls_certs %} [[tls.certificates]] - certFile = "/etc/traefik/tls/{{ cert_id }}_cert.pem" - keyFile = "/etc/traefik/tls/{{ cert_id }}_key.pem" + certFile = "/etc/ingress_proxy/tls/{{ cert_id }}_cert.pem" + keyFile = "/etc/ingress_proxy/tls/{{ cert_id }}_key.pem" {%- endfor %} {%- if traefik.tls_client_cert %} [tls.options.default.clientAuth] - caFiles = ["/etc/traefik/tls/traefik_client.crt"] + caFiles = ["/etc/ingress_proxy/tls/traefik_client.crt"] clientAuthType = "RequireAndVerifyClientCert" {%- endif %} diff --git a/infra/salt/salt/traefik/files/traefik.service b/infra/salt/salt/ingress_proxy/files/ingress-proxy.service similarity index 66% rename from infra/salt/salt/traefik/files/traefik.service rename to infra/salt/salt/ingress_proxy/files/ingress-proxy.service index fe2a885f30..1be388aa25 100644 --- a/infra/salt/salt/traefik/files/traefik.service +++ b/infra/salt/salt/ingress_proxy/files/ingress-proxy.service @@ -1,14 +1,14 @@ # See https://doc.traefik.io/traefik-enterprise/installing/on-premise/#systemd-linux-only [Unit] -Description=Traefik +Description=Traefik Ingress proxy After=network-online.target Wants=network-online.target systemd-networkd-wait-online.service [Service] -User=traefik -Group=traefik -ExecStart=/usr/bin/traefik +User=ingress_proxy +Group=ingress_proxy +ExecStart=/usr/bin/traefik --configFile=/etc/ingress_proxy/traefik.toml PrivateTmp=true PrivateDevices=false ProtectHome=true @@ -19,7 +19,7 @@ LimitNOFILE=32768 AmbientCapabilities=CAP_NET_BIND_SERVICE Restart=always RestartSec=2 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/traefik/files/traefik.toml.j2 b/infra/salt/salt/ingress_proxy/files/traefik.toml.j2 similarity index 93% rename from infra/salt/salt/traefik/files/traefik.toml.j2 rename to infra/salt/salt/ingress_proxy/files/traefik.toml.j2 index 6d7b0d9a07..6a92b3eb32 100644 --- a/infra/salt/salt/traefik/files/traefik.toml.j2 +++ b/infra/salt/salt/ingress_proxy/files/traefik.toml.j2 @@ -54,12 +54,12 @@ [log] level = "{{ pillar['rivet']['traefik']['log_level'] }}" - filePath = "/opt/traefik/traefik.log" + filePath = "/opt/ingress_proxy/traefik.log" format = "json" {% if pillar['rivet']['traefik']['access_logs'] %} [accessLog] - filePath = "/opt/traefik/access.log" + filePath = "/opt/ingress_proxy/access.log" format = "json" fields.defaultMode = "keep" fields.headers.defaultMode = "keep" @@ -74,7 +74,7 @@ [providers] [providers.file] - directory = "/etc/traefik/dynamic" + directory = "/etc/ingress_proxy/dynamic" {%- if 'consul-client' in grains['roles'] %} [providers.consulCatalog] diff --git a/infra/salt/salt/ingress_proxy/init.sls b/infra/salt/salt/ingress_proxy/init.sls new file mode 100644 index 0000000000..12f2f7914d --- /dev/null +++ b/infra/salt/salt/ingress_proxy/init.sls @@ -0,0 +1,127 @@ +{% set pool = grains['rivet']['pool_id'] %} +{% if pool == 'ing-px' or pool == 'local' %} + {% set tls_certs = ['cloudflare_rivet_gg'] %} + {% set tls_client_cert = true %} +{% elif pool == 'ing-job' %} + {% set tls_certs = ['letsencrypt_rivet_job'] %} + {% set tls_client_cert = false %} +{% endif %} + +# See on-prem installation instructions: https://doc.traefik.io/traefik-enterprise/installing/on-premise/#systemd-linux-only + +create_ingress_proxy_user: + user.present: + - name: ingress_proxy + - shell: /bin/false + - system: True + - usergroup: True + +create_etc_ingress_proxy: + file.directory: + - names: + - /etc/ingress_proxy/: {} + - /etc/ingress_proxy/dynamic/: {} + - /etc/ingress_proxy/tls/: {} + - /opt/ingress_proxy/: {} + - user: ingress_proxy + - group: ingress_proxy + - mode: 550 + - require: + - user: create_ingress_proxy_user + +push_etc_ingress_proxy: + file.managed: + - names: + # Static config + - /etc/ingress_proxy/traefik.toml: + - source: salt://ingress_proxy/files/traefik.toml.j2 + + # Dynamic configs + - /etc/ingress_proxy/dynamic/common.toml: + - source: salt://ingress_proxy/files/dynamic/common.toml.j2 + - /etc/ingress_proxy/dynamic/tls.toml: + - source: salt://ingress_proxy/files/dynamic/tls.toml.j2 + + # TLS certs + {%- for cert_id in tls_certs %} + - /etc/ingress_proxy/tls/{{ cert_id }}_cert.pem: + - contents: | + {{ pillar['tls']['tls_certs'][cert_id]['cert_pem'] | indent(12) }} + - /etc/ingress_proxy/tls/{{ cert_id }}_key.pem: + - contents: | + {{ pillar['tls']['tls_certs'][cert_id]['key_pem'] | indent(12) }} + {%- endfor %} + + # TLS client cert + {%- if tls_client_cert %} + - /etc/ingress_proxy/tls/traefik_client.crt: + - source: https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem + - source_hash: c1a581133a27f5fc98cbb1f32d048925470deab627e6b5f1b586f6df075d8493ec7d08ede04d0f31ec2c2cd74de27ed0df866e3874ad54a9e962695759ba7e5b + {%- endif %} + + - user: ingress_proxy + - group: ingress_proxy + - template: jinja + - context: + provider_http: + endpoint: 'https://route.api.{{ pillar['rivet']['domain']['base'] }}/v1/traefik/config?region={{grains['rivet']['region_id']}}&pool={{grains['rivet']['pool_id']}}&token={{pillar['rivet']['api-route']['token']}}' + {%- if pool == 'ing-px' %} + poll_interval: '2.5s' + {%- else %} + poll_interval: '1s' + {%- endif %} + namespace: {{ pillar['rivet']['namespace'] }} + pool: {{ pool }} + domain: {{ pillar['rivet']['domain'] }} + nebula_ipv4: {{ grains['nebula']['ipv4'] }} + traefik: + tls_certs: {{ tls_certs }} + tls_client_cert: {{ tls_client_cert }} + - mode: 440 + - require: + - file: create_etc_ingress_proxy + +push_ingress_proxy_service: + file.managed: + - name: /etc/systemd/system/ingress-proxy.service + - source: salt://ingress_proxy/files/ingress-proxy.service + - template: jinja + +# Stop and disable the old service under the old name first +stop_traefik_service: + service.dead: + - name: traefik + - enable: False + +# Manually restart the Traefik service yourself in order to prevent terminating +# connections needlessly +start_ingress_proxy_service: + service.running: + - name: ingress-proxy + - enable: True + - require: + - file: install_traefik + - file: push_ingress_proxy_service + - file: push_etc_ingress_proxy + - service: stop_traefik_service + +{%- if 'consul-client' in grains['roles'] %} +push_etc_consul_ingress_proxy_hcl: + file.managed: + - name: /etc/consul.d/ingress-proxy.hcl + - source: salt://ingress_proxy/files/consul/ingress-proxy.hcl.j2 + - template: jinja + - context: + service_name: {{ pool }} + nebula_ipv4: {{ grains['nebula']['ipv4'] }} + - require: + - file: create_etc_consul + cmd.run: + - name: consul reload + - require: + - service: start_consul_service + - file: push_etc_consul_ingress_proxy_hcl + - onchanges: + - file: push_etc_consul_ingress_proxy_hcl +{%- endif %} + diff --git a/infra/salt/salt/minio/files/minio.service b/infra/salt/salt/minio/files/minio.service index 42c5b8ffa7..bd2deb9294 100644 --- a/infra/salt/salt/minio/files/minio.service +++ b/infra/salt/salt/minio/files/minio.service @@ -18,7 +18,7 @@ LimitNOFILE=1048576 TasksMax=infinity TimeoutStopSec=infinity SendSIGKILL=no -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/nats/files/nats-server.service b/infra/salt/salt/nats/files/nats-server.service index b5e70460ae..e95cd18e2a 100644 --- a/infra/salt/salt/nats/files/nats-server.service +++ b/infra/salt/salt/nats/files/nats-server.service @@ -14,7 +14,7 @@ ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s SIGINT $MAINPID Restart=always RestartSec=2 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/node_exporter/files/node_exporter.service b/infra/salt/salt/node_exporter/files/node_exporter.service index 312919755d..227f0d3262 100644 --- a/infra/salt/salt/node_exporter/files/node_exporter.service +++ b/infra/salt/salt/node_exporter/files/node_exporter.service @@ -10,7 +10,7 @@ Type=simple ExecStart=/usr/bin/node_exporter --collector.cgroups --collector.network_route --collector.systemd Restart=always RestartSec=2 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/nomad/files/nomad.service.j2 b/infra/salt/salt/nomad/files/nomad.service.j2 index 7a3b88199e..cb746dc319 100644 --- a/infra/salt/salt/nomad/files/nomad.service.j2 +++ b/infra/salt/salt/nomad/files/nomad.service.j2 @@ -32,7 +32,7 @@ Restart=always RestartSec=2 TasksMax=infinity OOMScoreAdjust=-1000 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/prometheus/files/prometheus.service.j2 b/infra/salt/salt/prometheus/files/prometheus.service.j2 index c3b826e873..918be473b2 100644 --- a/infra/salt/salt/prometheus/files/prometheus.service.j2 +++ b/infra/salt/salt/prometheus/files/prometheus.service.j2 @@ -17,7 +17,7 @@ ExecStart=/usr/bin/prometheus \ ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=2 -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/prometheus/init.sls b/infra/salt/salt/prometheus/init.sls index 6a042b2428..0940db8022 100644 --- a/infra/salt/salt/prometheus/init.sls +++ b/infra/salt/salt/prometheus/init.sls @@ -3,23 +3,6 @@ {% set pool = grains['rivet']['pool_id'] %} -create_prometheus_user: - user.present: - - name: prometheus - - shell: /bin/false - - system: True - - usergroup: True - -mkdir_prometheus: - file.directory: - - names: - - /mnt/prometheus: - - user: prometheus - - group: prometheus - - mode: 700 - - require: - - user: create_prometheus_user - {% if grains['volumes']['prm']['mount'] %} {% set device = '/dev/disk/by-id/scsi-0Linode_Volume_' ~ grains['rivet']['name'] ~ '-prm' %} @@ -29,15 +12,37 @@ disk_create_prometheus: - fs_type: ext4 disk_mount_prometheus: + file.directory: + - name: /mnt/prometheus + - makedirs: True mount.mounted: - name: /mnt/prometheus - device: {{device}} - fstype: ext4 - require: - - file: mkdir_prometheus - blockdev: disk_create_prometheus {% endif %} +create_prometheus_user: + user.present: + - name: prometheus + - shell: /bin/false + - system: True + - usergroup: True + +mkdir_prometheus: + file.directory: + - names: + - /mnt/prometheus: + - user: prometheus + - group: prometheus + - mode: 700 + - require: + - user: create_prometheus_user + {%- if grains['volumes']['prm']['mount'] %} + - mount: disk_mount_prometheus + {%- endif %} + install_prometheus: archive.extracted: - name: /opt/prometheus_{{version}}/ diff --git a/infra/salt/salt/redis/files/redis.service.j2 b/infra/salt/salt/redis/files/redis.service.j2 index 281b363660..e743b0caec 100644 --- a/infra/salt/salt/redis/files/redis.service.j2 +++ b/infra/salt/salt/redis/files/redis.service.j2 @@ -14,7 +14,7 @@ ExecStartPre=-/usr/bin/docker stop {{container_name}} ExecStartPre=-/usr/bin/docker rm {{container_name}} ExecStartPre=/usr/bin/docker pull {{image}} ExecStart=/bin/sh -c '/usr/bin/docker run --rm --name {{container_name}} --user "$(id -u redis):$(id -g redis)" -v /etc/redis/{{db}}.conf:/etc/redis/{{db}}.conf {% if persistent %}-v /var/lib/redis/{{db}}/db:/var/lib/redis/{{db}}/db{% endif %} --network host -p "{{port}}:{{port}}" {{image}} /etc/redis/{{db}}.conf' -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/top.sls b/infra/salt/salt/top.sls index 572811fb94..163aff876a 100644 --- a/infra/salt/salt/top.sls +++ b/infra/salt/salt/top.sls @@ -32,7 +32,9 @@ base: - minio 'G@roles:traefik': - traefik - 'G@roles:traefik-cloudflare-proxy': - - traefik_cloudflare_proxy + 'G@roles:ingress-proxy': + - ingress_proxy + 'G@roles:cloudflare-proxy': + - cloudflare_proxy 'G@roles:docker-plugin-loki': - docker_plugin_loki diff --git a/infra/salt/salt/traefik/init.sls b/infra/salt/salt/traefik/init.sls index 3031c037d5..c545a3c05a 100644 --- a/infra/salt/salt/traefik/init.sls +++ b/infra/salt/salt/traefik/init.sls @@ -1,18 +1,7 @@ # https://github.com/traefik/traefik/releases {% set version = '2.10.4' %} -{% set pool = grains['rivet']['pool_id'] %} -{% if pool == 'ing-px' or pool == 'local' %} - {% set tls_certs = ['cloudflare_rivet_gg'] %} - {% set tls_client_cert = true %} -{% elif pool == 'ing-job' %} - {% set tls_certs = ['letsencrypt_rivet_job'] %} - {% set tls_client_cert = false %} -{% endif %} - -# See on-prem installation instructions: https://doc.traefik.io/traefik-enterprise/installing/on-premise/#systemd-linux-only - -create_traefik_exporter_user: +create_traefik_user: user.present: - name: traefik - shell: /bin/false @@ -34,6 +23,7 @@ install_traefik: - group: traefik - mode: 755 - require: + - user: create_traefik_user - archive: install_traefik cmd.run: - name: traefik version @@ -43,104 +33,17 @@ install_traefik: - onchanges: - file: install_traefik -create_etc_traefik: - file.directory: - - names: - - /etc/traefik/: {} - - /etc/traefik/dynamic/: {} - - /etc/traefik/tls/: {} - - /opt/traefik/: {} - - user: traefik - - group: traefik - - mode: 550 - -push_etc_traefik: - file.managed: - - names: - # Static config - - /etc/traefik/traefik.toml: - - source: salt://traefik/files/traefik.toml.j2 - - # Dynamic configs - - /etc/traefik/dynamic/common.toml: - - source: salt://traefik/files/dynamic/common.toml.j2 - - /etc/traefik/dynamic/tls.toml: - - source: salt://traefik/files/dynamic/tls.toml.j2 - - # TLS certs - {%- for cert_id in tls_certs %} - - /etc/traefik/tls/{{ cert_id }}_cert.pem: - - contents: | - {{ pillar['tls']['tls_certs'][cert_id]['cert_pem'] | indent(12) }} - - /etc/traefik/tls/{{ cert_id }}_key.pem: - - contents: | - {{ pillar['tls']['tls_certs'][cert_id]['key_pem'] | indent(12) }} - {%- endfor %} - - # TLS client cert - {%- if tls_client_cert %} - - /etc/traefik/tls/traefik_client.crt: - - source: https://developers.cloudflare.com/ssl/static/authenticated_origin_pull_ca.pem - - source_hash: c1a581133a27f5fc98cbb1f32d048925470deab627e6b5f1b586f6df075d8493ec7d08ede04d0f31ec2c2cd74de27ed0df866e3874ad54a9e962695759ba7e5b - {%- endif %} - - - user: traefik - - group: traefik - - template: jinja - - context: - provider_http: - endpoint: 'https://route.api.{{ pillar['rivet']['domain']['base'] }}/v1/traefik/config?region={{grains['rivet']['region_id']}}&pool={{grains['rivet']['pool_id']}}&token={{pillar['rivet']['api-route']['token']}}' - {%- if pool == 'ing-px' %} - poll_interval: '2.5s' - {%- else %} - poll_interval: '1s' - {%- endif %} - namespace: {{ pillar['rivet']['namespace'] }} - pool: {{ pool }} - domain: {{ pillar['rivet']['domain'] }} - nebula_ipv4: {{ grains['nebula']['ipv4'] }} - traefik: - tls_certs: {{ tls_certs }} - tls_client_cert: {{ tls_client_cert }} - - mode: 440 - - require: - - file: create_etc_traefik - - -push_traefik_service: - file.managed: - - name: /etc/systemd/system/traefik.service - - source: salt://traefik/files/traefik.service - - template: jinja - -# Manually restart the Traefik service yourself in order to prevent terminating -# connections needlessly -start_traefik_service: - service.running: - - name: traefik - - enable: True - - require: - - file: install_traefik - - file: push_traefik_service - - file: push_etc_traefik - {%- if 'consul-client' in grains['roles'] %} -push_etc_consul_traefik_hcl: - file.managed: +# Delete old Treafik Consul file that was moved +remove_etc_consul_traefik_hcl: + file.absent: - name: /etc/consul.d/traefik.hcl - - source: salt://traefik/files/consul/traefik.hcl.j2 - - template: jinja - - context: - service_name: {{ pool }} - nebula_ipv4: {{ grains['nebula']['ipv4'] }} - - require: - - file: create_etc_consul cmd.run: - name: consul reload - require: - service: start_consul_service - - file: push_etc_consul_traefik_hcl + - file: remove_etc_consul_traefik_hcl - onchanges: - - file: push_etc_consul_traefik_hcl + - file: remove_etc_consul_traefik_hcl {%- endif %} diff --git a/infra/salt/salt/traefik_cloudflare_proxy/init.sls b/infra/salt/salt/traefik_cloudflare_proxy/init.sls deleted file mode 100644 index c6464acfc3..0000000000 --- a/infra/salt/salt/traefik_cloudflare_proxy/init.sls +++ /dev/null @@ -1,54 +0,0 @@ -{% import_json "/srv/salt-context/rivet/secrets.json" as rivet_secrets %} - -create_traefik_cloudflare_proxy_user: - user.present: - - name: traefik_cloudflare_proxy - - shell: /bin/false - - system: True - - usergroup: True - -create_etc_traefik_cloudflare_proxy: - file.directory: - - names: - - /etc/traefik_cloudflare_proxy/: {} - - /etc/traefik_cloudflare_proxy/dynamic/: {} - - user: traefik_cloudflare_proxy - - group: traefik_cloudflare_proxy - - mode: 550 - -push_etc_traefik_cloudflare_proxy: - file.managed: - - names: - # Static config - - /etc/traefik_cloudflare_proxy/traefik.toml: - - source: salt://traefik_cloudflare_proxy/files/traefik.toml.j2 - - # Dynamic configs - - /etc/traefik_cloudflare_proxy/dynamic/common.toml: - - source: salt://traefik_cloudflare_proxy/files/dynamic/common.toml.j2 - - - user: traefik_cloudflare_proxy - - group: traefik_cloudflare_proxy - - template: jinja - - context: - loki_url: "{{ pillar['rivet']['logging']['loki']['endpoint'] }}" - access_client_id: "{{ pillar['cloudflare']['access']['proxy']['client_id'] }}" - access_client_secret: "{{ pillar['cloudflare']['access']['proxy']['client_secret'] }}" - - mode: 440 - - require: - - file: create_etc_traefik_cloudflare_proxy - -push_traefik_cloudflare_proxy_service: - file.managed: - - name: /etc/systemd/system/traefik_cloudflare_proxy.service - - source: salt://traefik_cloudflare_proxy/files/traefik_cloudflare_proxy.service - -# Manually restart the Traefik service yourself in order to prevent terminating -# connections needlessly -start_traefik_cloudflare_proxy_service: - service.running: - - name: traefik_cloudflare_proxy - - enable: True - - require: - - file: push_traefik_cloudflare_proxy_service - - file: push_etc_traefik_cloudflare_proxy diff --git a/infra/salt/salt/traffic_server/files/consul/traffic_server.hcl.j2 b/infra/salt/salt/traffic_server/files/consul/traffic_server.hcl.j2 index 6337cdc181..d0df35a654 100644 --- a/infra/salt/salt/traffic_server/files/consul/traffic_server.hcl.j2 +++ b/infra/salt/salt/traffic_server/files/consul/traffic_server.hcl.j2 @@ -42,24 +42,23 @@ services { # and https://imagekit.io/blog/ultimate-guide-to-http-caching-for-static-assets/ "{{prefix}}.http.middlewares.ats-cdn-cache-control.headers.customResponseHeaders.Cache-Control=public, max-age=604800, immutable", - # MARK: routers.ats-build - # "{{prefix}}.http.routers.ats-build.entryPoints=lb-443", - # "{{prefix}}.http.routers.ats-build.rule=Host(`cdn.{{domain['base']}}`) && PathPrefix(`/build/`)", - # "{{prefix}}.http.routers.ats-build.middlewares=ats-build-auth, ats-build-path, ats-cdn-lfs", - # "{{prefix}}.http.routers.ats-build.tls=true", - - # MARK: middlewares.ats-build-auth - # TODO: Add more secure per-game authentication with https://doc.traefik.io/traefik/middlewares/http/forwardauth/ - # "{{prefix}}.http.middlewares.ats-build-auth.basicAuth.users=job_run_image:$apr1$0peMM8t1$/ST0ckxxbp4AY.SUzkk4E1", - # Without this, we'll receive a `SignatureDoesNotMatch` error - # "{{prefix}}.http.middlewares.ats-build-auth.basicAuth.removeHeader=true", - - # MARK: middlewares.ats-build-path - # "{{prefix}}.http.middlewares.ats-build-path.replacePathRegex.regex=^/build/(.*)", - # "{{prefix}}.http.middlewares.ats-build-path.replacePathRegex.replacement=/s3-cache/{{namespace}}-bucket-build/${1}", + {% for provider, _ in s3_providers.items() %} + {% for bucket in ["job-log", "team-billing", "lobby-history-export", "nomad-log-export"] %} + # MARK: http.routers-{{provider}}-{{bucket}} + "{{prefix}}.http.routers.ats-{{provider}}-{{bucket}}.entryPoints=lb-443", + "{{prefix}}.http.routers.ats-{{provider}}-{{bucket}}.rule=Host(`cdn.{{domain['base']}}`) && PathPrefix(`/{{provider}}/{{bucket}}/`)", + "{{prefix}}.http.routers.ats-{{provider}}-{{bucket}}.middlewares=ats-{{bucket}}-path, ats-cors, ats-cdn", + "{{prefix}}.http.routers.ats-{{provider}}-{{bucket}}.tls=true", + + # MARK: middlewares.ats-{{provider}}-{{bucket}}-path + "{{prefix}}.http.middlewares.ats-{{provider}}-{{bucket}}-path.replacePathRegex.regex=^/{{provider}}/{{bucket}}/(.*)", + "{{prefix}}.http.middlewares.ats-{{provider}}-{{bucket}}-path.replacePathRegex.replacement=/s3-cache/{{provider}}/{{namespace}}-bucket-{{bucket}}/${1}", + {% endfor %} + {% endfor %} {% for bucket in ["job-log", "team-billing", "lobby-history-export", "nomad-log-export"] %} # MARK: http.routers-{{bucket}} + # Default route for backwards compatibility "{{prefix}}.http.routers.ats-{{bucket}}.entryPoints=lb-443", "{{prefix}}.http.routers.ats-{{bucket}}.rule=Host(`cdn.{{domain['base']}}`) && PathPrefix(`/{{bucket}}/`)", "{{prefix}}.http.routers.ats-{{bucket}}.middlewares=ats-{{bucket}}-path, ats-cors, ats-cdn", diff --git a/infra/salt/salt/traffic_server/files/etc/dynamic/remap.config.j2 b/infra/salt/salt/traffic_server/files/etc/dynamic/remap.config.j2 index 6f3d1a6157..ab80114b21 100644 --- a/infra/salt/salt/traffic_server/files/etc/dynamic/remap.config.j2 +++ b/infra/salt/salt/traffic_server/files/etc/dynamic/remap.config.j2 @@ -1,2 +1,7 @@ -map /s3-cache {{s3_endpoint}} @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v4.config +{%- for provider_name, provider in s3_providers.items() %} +map /s3-cache/{{provider_name}} {{provider['endpoint_internal']}} @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v4_{{provider_name}}.config +{%- endfor %} +{%- if 'default' in s3_providers %} +map /s3-cache {{s3_providers['default']['endpoint_internal']}} @plugin=s3_auth.so @pparam=--config @pparam=s3_auth_v4_default.config +{%- endif %} diff --git a/infra/salt/salt/traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 b/infra/salt/salt/traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 index 784fe57e17..c7a626b3b7 100644 --- a/infra/salt/salt/traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 +++ b/infra/salt/salt/traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 @@ -2,4 +2,4 @@ access_key={{s3_access_key_id}} secret_key={{s3_secret_access_key}} version=4 -v4-region-map=s3_region_map.config +v4-region-map={{s3_region_map_file_name}}.config diff --git a/infra/salt/salt/traffic_server/files/etc/dynamic/s3_region_map.config.j2 b/infra/salt/salt/traffic_server/files/etc/dynamic/s3_region_map.config.j2 index a232ea92fc..331ede7ef4 100644 --- a/infra/salt/salt/traffic_server/files/etc/dynamic/s3_region_map.config.j2 +++ b/infra/salt/salt/traffic_server/files/etc/dynamic/s3_region_map.config.j2 @@ -1,5 +1,5 @@ -{% set s3_host = s3_endpoint.split('://')[1] if s3_endpoint is not none and '://' in s3_endpoint else s3_endpoint %} -{% set s3_hostname = s3_host.split(':')[0] if s3_host is not none and ':' in s3_host else s3_host %} +{% set s3_host = s3_endpoint.split('://')[1] if '://' in s3_endpoint else s3_endpoint %} +{% set s3_hostname = s3_host.split(':')[0] if s3_host and ':' in s3_host else s3_host %} # Default region {{s3_hostname}}: {{s3_region}} diff --git a/infra/salt/salt/traffic_server/files/trafficserver.service b/infra/salt/salt/traffic_server/files/trafficserver.service index 3cd4a9ba15..f6e3cb85d9 100644 --- a/infra/salt/salt/traffic_server/files/trafficserver.service +++ b/infra/salt/salt/traffic_server/files/trafficserver.service @@ -6,11 +6,11 @@ After=network-online.target Type=simple User=trafficserver Group=trafficserver -EnvironmentFile=-/etc/default/trafficserver +Environment="TM_START=yes" PIDFile=/run/trafficserver/manager.lock ExecStart=/var/rivet-nix/result/traffic_server/bin/traffic_manager $TM_DAEMON_ARGS ExecReload=/var/rivet-nix/result/traffic_server/bin/traffic_manager config reload -{% if pillar['rivet']['deploy']['local']['restrict_service_resources'] %} +{% if 'local' in pillar.rivet.deploy and pillar.rivet.deploy.local.restrict_service_resources %} Nice=10 CPUAffinity=0 {% endif %} diff --git a/infra/salt/salt/traffic_server/init.sls b/infra/salt/salt/traffic_server/init.sls index b7c9eb6cb3..ce5dbaba32 100644 --- a/infra/salt/salt/traffic_server/init.sls +++ b/infra/salt/salt/traffic_server/init.sls @@ -8,7 +8,7 @@ disk_create_traffic_serer: disk_mount_traffic_server: file.directory: - name: /mnt/trafficserver - + - makedirs: True mount.mounted: - name: /mnt/trafficserver - device: {{ device }} @@ -26,31 +26,23 @@ create_trafficserver_user: create_mnt_db_trafficserver: file.directory: - - name: /mnt/trafficserver/db - - user: trafficserver - - group: trafficserver - - mode: 700 - - makedirs: True - {%- if grains['volumes']['ats']['mount'] %} - - require: - - mount: disk_mount_traffic_server - {% endif %} - -create_var_log_trafficserver: - file.directory: - - name: /var/log/trafficserver + - names: + - /mnt/trafficserver/db + - /var/log/trafficserver + - /run/trafficserver - user: trafficserver - group: trafficserver - mode: 700 - makedirs: True - {%- if grains['volumes']['ats']['mount'] %} - require: + - user: create_trafficserver_user + {%- if grains['volumes']['ats']['mount'] %} - mount: disk_mount_traffic_server - {% endif %} + {%- endif %} push_trafficserver_service: file.managed: - - name: /lib/systemd/system/trafficserver.service + - name: /etc/systemd/system/trafficserver.service - source: salt://traffic_server/files/trafficserver.service - template: jinja - onchanges: @@ -62,7 +54,6 @@ start_trafficserver_service: - require: - file: create_mnt_db_trafficserver - file: push_trafficserver_service - - file: create_var_log_trafficserver - onchanges: - file: push_trafficserver_service @@ -70,10 +61,14 @@ push_etc_trafficserver_static: file.recurse: - name: /etc/trafficserver/ - source: salt://traffic_server/files/etc/static/ + - user: trafficserver + - group: trafficserver - file_mode: 644 - dir_mode: 755 # Keep other files, since we'll also be writing files in push_etc_trafficserver_dynamic - clean: False + - require: + - user: create_trafficserver_user push_etc_trafficserver_dynamic: file.managed: @@ -82,21 +77,42 @@ push_etc_trafficserver_dynamic: - source: salt://traffic_server/files/etc/dynamic/records.config.j2 - /etc/trafficserver/remap.config: - source: salt://traffic_server/files/etc/dynamic/remap.config.j2 - - /etc/trafficserver/s3_auth_v4.config: - - source: salt://traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 - - /etc/trafficserver/s3_region_map.config: - - source: salt://traffic_server/files/etc/dynamic/s3_region_map.config.j2 - - /etc/trafficserver/stored.config: + - /etc/trafficserver/storage.config: - source: salt://traffic_server/files/etc/dynamic/storage.config.j2 + - user: trafficserver + - group: trafficserver - mode: 644 - template: jinja - context: nebula_ipv4: {{ grains['nebula']['ipv4'] }} - s3_endpoint: {{ pillar['s3']['config']['default']['endpoint_internal'] }} - s3_region: {{ pillar['s3']['config']['default']['region'] }} - s3_access_key_id: {{ pillar['s3']['access']['default']['persistent_access_key_id'] }} - s3_secret_access_key: {{ pillar['s3']['access']['default']['persistent_access_key_secret'] }} + s3_providers: {{ pillar['s3']['config'] }} volume_size_cache: {{ grains['volumes']['ats']['size']|int - 1 }}G + - require: + - file: push_etc_trafficserver_static + - user: create_trafficserver_user + +{%- for provider, _ in pillar['s3']['config'].items() %} +push_etc_trafficserver_dynamic_{{provider}}: + file.managed: + - names: + - /etc/trafficserver/s3_auth_v4_{{provider}}.config: + - source: salt://traffic_server/files/etc/dynamic/s3_auth_v4.config.j2 + - /etc/trafficserver/s3_region_map_{{provider}}.config: + - source: salt://traffic_server/files/etc/dynamic/s3_region_map.config.j2 + - user: trafficserver + - group: trafficserver + - mode: 644 + - template: jinja + - context: + s3_endpoint: {{ pillar['s3']['config'][provider]['endpoint_internal'] }} + s3_region: {{ pillar['s3']['config'][provider]['region'] }} + s3_access_key_id: {{ pillar['s3']['access'][provider]['persistent_access_key_id'] }} + s3_secret_access_key: {{ pillar['s3']['access'][provider]['persistent_access_key_secret'] }} + s3_region_map_file_name: s3_region_map_{{provider}} + - require: + - file: push_etc_trafficserver_static + - user: create_trafficserver_user +{%- endfor %} reload_traffic_server_config: cmd.run: @@ -105,9 +121,15 @@ reload_traffic_server_config: - cmd: start_trafficserver_service - file: push_etc_trafficserver_static - file: push_etc_trafficserver_dynamic + {%- for provider, _ in pillar['s3']['config'].items() %} + - push_etc_trafficserver_dynamic_{{provider}} + {%- endfor %} - onchanges: - file: push_etc_trafficserver_static - file: push_etc_trafficserver_dynamic + {%- for provider, _ in pillar['s3']['config'].items() %} + - push_etc_trafficserver_dynamic_{{provider}} + {%- endfor %} push_etc_consul_traffic_server_hcl: file.managed: @@ -118,6 +140,7 @@ push_etc_consul_traffic_server_hcl: namespace: {{ pillar['rivet']['namespace'] }} domain: {{ pillar['rivet']['domain'] }} nebula_ipv4: {{ grains['nebula']['ipv4'] }} + s3_providers: {{ pillar['s3']['config'] }} - require: - file: create_etc_consul cmd.run: diff --git a/infra/tests/standalone/main.tf b/infra/tests/standalone/main.tf index 953cac3c70..25906539a4 100644 --- a/infra/tests/standalone/main.tf +++ b/infra/tests/standalone/main.tf @@ -43,19 +43,21 @@ module "server" { inbound_ipv4_cidr = ["0.0.0.0/0"] inbound_ipv6_cidr = ["::/0"] }, + + # TODO: Nebula IPv6 is theoretically supported (https://nebula.defined.net/docs/config/preferred-ranges/#how-nebula-orders-underlay-ip-addresses-it-learns-about), but seems to be problematic { label = "nebula-udp" ports = "4242" protocol = "udp" inbound_ipv4_cidr = ["0.0.0.0/0"] - inbound_ipv6_cidr = ["::/0"] + inbound_ipv6_cidr = [] }, { label = "nebula-tcp" ports = "4242" protocol = "tcp" inbound_ipv4_cidr = ["0.0.0.0/0"] - inbound_ipv6_cidr = ["::/0"] + inbound_ipv6_cidr = [] }, ] } diff --git a/infra/tests/stress-param/job-param.nomad b/infra/tests/stress-param/job-param.nomad index 3701abf2ec..ac234ff8b4 100644 --- a/infra/tests/stress-param/job-param.nomad +++ b/infra/tests/stress-param/job-param.nomad @@ -45,7 +45,7 @@ job "test-param" { } artifact { - source = "https://assets.rivet.gg/graphics/landing.png" + source = "https://assets2.rivet.gg/graphics/landing.png" destination = "local/landing.png" } diff --git a/infra/tf/master_cluster/nebula_lighthouse.tf b/infra/tf/master_cluster/nebula_lighthouse.tf index f65205bde6..5c5e988f99 100644 --- a/infra/tf/master_cluster/nebula_lighthouse.tf +++ b/infra/tf/master_cluster/nebula_lighthouse.tf @@ -25,20 +25,21 @@ module "nebula_lighthouse_server" { ] backup = true + # Nebula does not support IPv6 firewall_inbound = [ { label = "nebula-udp" ports = "4242" protocol = "udp" inbound_ipv4_cidr = ["0.0.0.0/0"] - inbound_ipv6_cidr = ["::/0"] + inbound_ipv6_cidr = [] }, { label = "nebula-tcp" ports = "4242" protocol = "tcp" inbound_ipv4_cidr = ["0.0.0.0/0"] - inbound_ipv6_cidr = ["::/0"] + inbound_ipv6_cidr = [] }, ] } diff --git a/infra/tf/modules/install_nebula/main.tf b/infra/tf/modules/install_nebula/main.tf index ad22969d6f..0066351257 100644 --- a/infra/tf/modules/install_nebula/main.tf +++ b/infra/tf/modules/install_nebula/main.tf @@ -18,6 +18,16 @@ locals { lighthouse = { am_lighthouse = var.is_lighthouse hosts = var.is_lighthouse ? null : sort(var.lighthouse_hosts) + + # Disable IPv6 + remote_allow_list = { + "0.0.0.0/0" = true + "::/0" = false + } + local_allow_list = { + "0.0.0.0/0" = true + "::/0" = false + } } listen = { host = "0.0.0.0" diff --git a/infra/tf/modules/secrets/main.tf b/infra/tf/modules/secrets/main.tf index 6fbccc4dec..4124127005 100644 --- a/infra/tf/modules/secrets/main.tf +++ b/infra/tf/modules/secrets/main.tf @@ -10,6 +10,6 @@ terraform { data "external" "bolt_secret" { for_each = var.keys - program = var.optional ? ["bolt", "secret", "get", "--format=json", "--optional", each.key] : ["bolt", "secret", "get", "--format=json", each.key] + program = ["${path.module}/scripts/get_secret.sh", each.key, var.optional ? "true" : "false"] } diff --git a/infra/tf/modules/secrets/scripts/get_secret.sh b/infra/tf/modules/secrets/scripts/get_secret.sh new file mode 100755 index 0000000000..7558d481ff --- /dev/null +++ b/infra/tf/modules/secrets/scripts/get_secret.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euf + +export BOLT_HEADLESS=1 + +KEY="$1" +OPTIONAL="$2" + +if [ "$OPTIONAL" == "true" ]; then + bolt secret get --format=json --optional "$KEY" +else + bolt secret get --format=json "$KEY" +fi + diff --git a/infra/tf/nomad/files/imagor/imagor.nomad.tpl b/infra/tf/nomad/files/imagor/imagor.nomad.tpl index 2ce0c2dcc8..461582a088 100644 --- a/infra/tf/nomad/files/imagor/imagor.nomad.tpl +++ b/infra/tf/nomad/files/imagor/imagor.nomad.tpl @@ -75,14 +75,41 @@ job "imagor:${dc}" { "${prefix}.http.middlewares.imagor-cdn-cache-control.headers.customResponseHeaders.Cache-Control=public, max-age=604800, immutable", # "${prefix}.http.middlewares.imagor-cdn-cache-control.headers.customResponseHeaders.Vary=Accept", + # Individual S3 providers + %{ for preset in imagor_presets } + %{ for provider in s3_providers } + # routers.imagor-${provider}-${preset.key} + "${prefix}.http.routers.imagor-${provider}-${preset.key}.entrypoints=lb-443", + "${prefix}.http.routers.imagor-${provider}-${preset.key}.priority=${preset.priority}", + %{ if preset.query != null } + "${prefix}.http.routers.imagor-${provider}-${preset.key}.rule=HostRegexp(`media.${shared.domain.base}`, `media.{region:.+}.${shared.domain.base}`) && Path(`/${provider}${preset.path}`) && Query(%{ for x in preset.query }`${x[0]}=${x[1]}`,%{ endfor })", + %{ else } + "${prefix}.http.routers.imagor-${provider}-${preset.key}.rule=HostRegexp(`media.${shared.domain.base}`, `media.{region:.+}.${shared.domain.base}`) && Path(`/${provider}${preset.path}`)", + %{ endif } + + %{ if preset.game_cors } + "${prefix}.http.routers.imagor-${provider}-${preset.key}.middlewares=imagor-${provider}-${preset.key}-path, imagor-cors-game, imagor-cdn", + %{ else } + "${prefix}.http.routers.imagor-${provider}-${preset.key}.middlewares=imagor-${provider}-${preset.key}-path, imagor-cors, imagor-cdn", + %{ endif } + "${prefix}.http.routers.imagor-${provider}-${preset.key}.tls=true", + + # middlewares.imagor-${provider}-${preset.key}-path + "${prefix}.http.middlewares.imagor-${provider}-${preset.key}-path.replacePathRegex.regex=/${provider}${preset.path_regexp}", + # Escape replacement and replace `s3-cache/` with `s3-cache/{provider}/` + "${prefix}.http.middlewares.imagor-${provider}-${preset.key}-path.replacePathRegex.replacement=${replace(replace(preset.path_regex_replacement, "$${", "$$${"), "s3-cache%2F", "s3-cache%2F${provider}%2F")}", + %{ endfor } + %{ endfor } + + # Default S3 provider %{ for preset in imagor_presets } # routers.imagor-${preset.key} "${prefix}.http.routers.imagor-${preset.key}.entrypoints=lb-443", "${prefix}.http.routers.imagor-${preset.key}.priority=${preset.priority}", %{ if preset.query != null } - "${prefix}.http.routers.imagor-${preset.key}.rule=(Host(`media.${shared.domain.base}`) || HostRegexp(`media.{region:.+}.${shared.domain.base}`)) && Path(`${preset.path}`) && Query(%{ for x in preset.query }`${x[0]}=${x[1]}`,%{ endfor })", + "${prefix}.http.routers.imagor-${preset.key}.rule=HostRegexp(`media.${shared.domain.base}`, `media.{region:.+}.${shared.domain.base}`) && Path(`${preset.path}`) && Query(%{ for x in preset.query }`${x[0]}=${x[1]}`,%{ endfor })", %{ else } - "${prefix}.http.routers.imagor-${preset.key}.rule=(Host(`media.${shared.domain.base}`) || HostRegexp(`media.{region:.+}.${shared.domain.base}`)) && Path(`${preset.path}`)", + "${prefix}.http.routers.imagor-${preset.key}.rule=HostRegexp(`media.${shared.domain.base}`, `media.{region:.+}.${shared.domain.base}`) && Path(`${preset.path}`)", %{ endif } %{ if preset.game_cors } "${prefix}.http.routers.imagor-${preset.key}.middlewares=imagor-${preset.key}-path, imagor-cors-game, imagor-cdn", diff --git a/infra/tf/nomad/imagor.tf b/infra/tf/nomad/imagor.tf index 319863e5ec..16d969e2ee 100644 --- a/infra/tf/nomad/imagor.tf +++ b/infra/tf/nomad/imagor.tf @@ -31,5 +31,6 @@ resource "nomad_job" "imagor" { result_storage_s3_access_key_id = var.s3_persistent_access_key_id result_storage_s3_secret_access_key = nonsensitive(var.s3_persistent_access_key_secret) result_storage_s3_bucket = "${var.namespace}-bucket-imagor-result-storage" + s3_providers = var.s3_providers }) } diff --git a/infra/tf/nomad/vars.tf b/infra/tf/nomad/vars.tf index 48030d5035..e080e3dccb 100644 --- a/infra/tf/nomad/vars.tf +++ b/infra/tf/nomad/vars.tf @@ -67,3 +67,6 @@ variable "s3_persistent_access_key_secret" { sensitive = true } +variable "s3_providers" { + type = list(string) +} diff --git a/infra/tf/s3_backblaze/buckets.tf b/infra/tf/s3_backblaze/buckets.tf index 3e9a69c8b2..366bcac749 100644 --- a/infra/tf/s3_backblaze/buckets.tf +++ b/infra/tf/s3_backblaze/buckets.tf @@ -1,6 +1,11 @@ resource "b2_bucket" "bucket" { for_each = var.s3_buckets + lifecycle { + # TODO: Remove this. Fixes an issue with accidentally enabled file locks that can't be disabled. + ignore_changes = ["file_lock_configuration"] + } + bucket_name = each.key bucket_type = "allPrivate" diff --git a/lib/api-helper/build/Cargo.toml b/lib/api-helper/build/Cargo.toml index 48b3040eda..36a4cf56f9 100644 --- a/lib/api-helper/build/Cargo.toml +++ b/lib/api-helper/build/Cargo.toml @@ -37,7 +37,7 @@ serde = { version = "1.0", features = ["derive"] } serde_array_query = { git = "https://github.com/rivet-gg/serde_array_query.git", rev = "b9f8bfad77aea6f01dccc6cb77146b8c5daecaa3" } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } types = { path = "../../types/core" } diff --git a/lib/api-helper/build/src/start.rs b/lib/api-helper/build/src/start.rs index 67c32b0d0e..6c9d67bbee 100644 --- a/lib/api-helper/build/src/start.rs +++ b/lib/api-helper/build/src/start.rs @@ -5,6 +5,8 @@ use hyper::{ Body, Request, Response, Server, }; use std::{convert::Infallible, future::Future, net::SocketAddr, sync::Arc, time::Instant}; +use tokio::sync::Semaphore; +use tokio::time::{sleep, Duration}; use tracing::Instrument; use uuid::Uuid; @@ -66,9 +68,13 @@ where .and_then(|v| v.parse::().ok()) .unwrap(); + // let semaphore = Arc::new(Semaphore::new(8)); + // A `MakeService` that produces a `Service` to handle each connection let health_check_config = Arc::new(health_check_config); let make_service = make_service_fn(move |conn: &AddrStream| { + // let semaphore = semaphore.clone(); + let shared_client = shared_client.clone(); let pools = pools.clone(); let cache = cache.clone(); @@ -79,6 +85,8 @@ where let service = service_fn(move |req: Request| { let start = Instant::now(); + // let semaphore = semaphore.clone(); + let shared_client = shared_client.clone(); let pools = pools.clone(); let cache = cache.clone(); @@ -88,28 +96,71 @@ where let ray_id = Uuid::new_v4(); let req_span = tracing::info_span!("http request", method = %req.method(), uri = %req.uri(), %ray_id); async move { + let sem_start = Instant::now(); + // let _semaphor_acquire = semaphore.acquire().await.unwrap(); + // let semaphor_duration = sem_start.elapsed(); + tracing::info!( method = %req.method(), uri = %req.uri(), headers = ?req.headers(), body_size_hint = ?req.body().size_hint(), remote_addr = %remote_addr, + // semaphor_duration = ?semaphor_duration, "http request meta" ); - let res: Response = - match rivet_health_checks::handle(&*health_check_config, req).await { - Ok(res) => res, - Err(req) => { - let mut response = handle(shared_client, pools, cache, ray_id, req) - .instrument(tracing::info_span!("request_handle")) - .await?; - response - .headers_mut() - .insert("rvt-ray-id", ray_id.to_string().parse()?); - response + let res = tokio::task::Builder::new() + .name("api_helper::handle") + .spawn(async move { + // let res = Response::builder() + // .status(http::StatusCode::OK) + // .body(Body::from("ok"))?; + + let res = match rivet_health_checks::handle(&*health_check_config, req) + .await + { + Ok(res) => res, + Err(req) => { + // Response::builder() + // .status(http::StatusCode::OK) + // .body(Body::from("ok"))? + + let mut response = handle(shared_client, pools, cache, ray_id, req) + .instrument(tracing::info_span!("request_handle")) + .await?; + response + .headers_mut() + .insert("rvt-ray-id", ray_id.to_string().parse()?); + response + } + }; + + Result::, http::Error>::Ok(res) + }); + let res = match res { + Ok(res) => match res.await { + Ok(res) => match res { + Ok(res) => res, + Err(err) => { + tracing::error!(?err, "http error"); + return Err(err); + } + }, + Err(_) => { + tracing::error!("http error"); + return Ok(Response::builder() + .status(http::StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty())?); } - }; + }, + Err(err) => { + tracing::error!(?err, "http error"); + return Ok(Response::builder() + .status(http::StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty())?); + } + }; if res.status().is_server_error() { tracing::error!(status = ?res.status().as_u16(), "http server error"); diff --git a/lib/bolt/Cargo.lock b/lib/bolt/Cargo.lock index c78a3a3d18..b0f8597311 100644 --- a/lib/bolt/Cargo.lock +++ b/lib/bolt/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -452,6 +467,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -1031,6 +1061,12 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "global-error" version = "0.1.5" @@ -1450,6 +1486,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.8" @@ -1520,6 +1565,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -2048,6 +2102,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2499,11 +2559,12 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", "mio", diff --git a/lib/bolt/cli/Cargo.toml b/lib/bolt/cli/Cargo.toml index 8e5e0adee7..e4f2c37958 100644 --- a/lib/bolt/cli/Cargo.toml +++ b/lib/bolt/cli/Cargo.toml @@ -13,4 +13,4 @@ clap = { version = "4.3", features = ["derive", "env"] } duct = "0.13" serde_json = "1.0" tempfile = "3.2" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } diff --git a/lib/bolt/cli/src/commands/config.rs b/lib/bolt/cli/src/commands/config.rs index ce7db62a25..9571fda88f 100644 --- a/lib/bolt/cli/src/commands/config.rs +++ b/lib/bolt/cli/src/commands/config.rs @@ -21,6 +21,12 @@ pub enum SubCommand { /// Adds missing regions from supported cloud providers to default_regions.toml. #[clap(hide(true))] GenerateDefaultRegions, + ServiceDependencies { + #[clap(index = 1)] + svc_name: String, + #[clap(long, short = 'r')] + recursive: bool, + }, } impl SubCommand { @@ -34,6 +40,29 @@ impl SubCommand { tasks::config::set_namespace(&namespace).await?; } Self::GenerateDefaultRegions => tasks::config::generate_default_regions().await?, + Self::ServiceDependencies { + svc_name, + recursive, + } => { + // Build project + let ctx = bolt_core::context::ProjectContextData::new( + std::env::var("BOLT_NAMESPACE").ok(), + ) + .await; + + // Read deps + let deps = if *recursive { + ctx.recursive_dependencies(&[&svc_name]).await + } else { + let svc = ctx.service_with_name(svc_name).await; + svc.dependencies().await + }; + + // Print deps + for dep in deps { + println!("{}", dep.name()); + } + } } Ok(()) diff --git a/lib/bolt/cli/src/commands/db/migrate.rs b/lib/bolt/cli/src/commands/db/migrate.rs index d95808668f..548af74e08 100644 --- a/lib/bolt/cli/src/commands/db/migrate.rs +++ b/lib/bolt/cli/src/commands/db/migrate.rs @@ -21,6 +21,8 @@ pub enum SubCommand { /// /// Helpful when dealing with bugged migrations. Force { service: String, num: usize }, + /// Lists migrations + List, /// Drops the entire database. Drop { service: String }, } @@ -61,6 +63,11 @@ impl SubCommand { Self::Force { service, num } => { tasks::migrate::force(&ctx, &ctx.service_with_name(&service).await, num).await?; } + Self::List => { + for svc in ctx.services_with_migrations().await { + println!("{}", svc.name()); + } + } Self::Drop { service } => { tasks::migrate::drop(&ctx, &ctx.service_with_name(&service).await).await?; } diff --git a/lib/bolt/cli/src/commands/infra.rs b/lib/bolt/cli/src/commands/infra.rs index 1f68472b5d..e9897f2cfd 100644 --- a/lib/bolt/cli/src/commands/infra.rs +++ b/lib/bolt/cli/src/commands/infra.rs @@ -8,16 +8,23 @@ use clap::Parser; #[derive(Parser)] pub enum SubCommand { /// Prints out the plan used to provision the Rivet cluster. - Plan, + Plan { + #[clap(long)] + start_at: Option, + }, /// Provisions all infrastructure required for the Rivet cluster. Up { #[clap(long, short = 'y')] yes: bool, + #[clap(long)] + start_at: Option, }, /// Destroys all provisioned infrastructure. Destroy { #[clap(long, short = 'y')] yes: bool, + #[clap(long)] + start_at: Option, }, /// Manages infrastructure migrations. #[clap(hide(true))] @@ -35,16 +42,24 @@ pub enum MigrateSubCommand { impl SubCommand { pub async fn execute(self, ctx: ProjectContext) -> Result<()> { match self { - Self::Plan => { - let plan = tasks::infra::build_plan(&ctx)?; - println!("{plan:#?}"); + Self::Plan { start_at } => { + let plan = tasks::infra::build_plan(&ctx, start_at)?; + for step in plan { + println!("{}: {:?}", step.name_id, step.kind); + } } - Self::Up { yes: auto_approve } => { - let plan = tasks::infra::build_plan(&ctx)?; + Self::Up { + yes: auto_approve, + start_at, + } => { + let plan = tasks::infra::build_plan(&ctx, start_at)?; tasks::infra::execute_plan(&ctx, &plan, ExecutePlanOpts { auto_approve }).await?; } - Self::Destroy { yes: auto_approve } => { - let plan = tasks::infra::build_plan(&ctx)?; + Self::Destroy { + yes: auto_approve, + start_at, + } => { + let plan = tasks::infra::build_plan(&ctx, start_at)?; tasks::infra::destroy_plan(&ctx, &plan, ExecutePlanOpts { auto_approve }).await?; } Self::Migrate { command } => match command { diff --git a/lib/bolt/cli/src/commands/init.rs b/lib/bolt/cli/src/commands/init.rs index ae0a69179a..a94c2b334a 100644 --- a/lib/bolt/cli/src/commands/init.rs +++ b/lib/bolt/cli/src/commands/init.rs @@ -26,7 +26,7 @@ impl InitOpts { let ctx = context::ProjectContextData::new(Some(self.namespace.clone())).await; // Apply infra - let plan = tasks::infra::build_plan(&ctx)?; + let plan = tasks::infra::build_plan(&ctx, None)?; tasks::infra::execute_plan( &ctx, &plan, diff --git a/lib/bolt/cli/src/main.rs b/lib/bolt/cli/src/main.rs index 454d5466b2..4a45be3b52 100644 --- a/lib/bolt/cli/src/main.rs +++ b/lib/bolt/cli/src/main.rs @@ -134,6 +134,10 @@ async fn main() -> Result { fn prompt_prod() -> Result<()> { use std::io::Write; + if std::env::var("BOLT_HEADLESS").ok() == Some("1".to_string()) { + return Ok(()); + } + let mut input = String::new(); print!("Are you sure you want to run this command in prod? (yes) "); diff --git a/lib/bolt/config/default_regions.toml b/lib/bolt/config/default_regions.toml index e422e62313..56c7bf7e69 100644 --- a/lib/bolt/config/default_regions.toml +++ b/lib/bolt/config/default_regions.toml @@ -7,6 +7,7 @@ provider = "linode" provider_region = "us-southeast" netnum = 0 preferred_subnets = ["192.168.128.0/17"] +supports_vlan = true [lnd-sfo] id = "19f1b737-0ff1-4676-a834-1af8c4e9890f" diff --git a/lib/bolt/core/Cargo.toml b/lib/bolt/core/Cargo.toml index 491a0cddae..ae063eda33 100644 --- a/lib/bolt/core/Cargo.toml +++ b/lib/bolt/core/Cargo.toml @@ -34,7 +34,7 @@ serde = { version ="1.0", features = ["derive"] } serde_json = "1.0" tempfile = "3.2" thiserror = "1.0" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } toml = "0.7" toml_edit = "0.19" types = { path = "../../types/core" } diff --git a/lib/bolt/core/src/context/project.rs b/lib/bolt/core/src/context/project.rs index ac469f523e..9e61ed6f08 100644 --- a/lib/bolt/core/src/context/project.rs +++ b/lib/bolt/core/src/context/project.rs @@ -112,7 +112,7 @@ impl ProjectContextData { // Assign references to all services after we're done for svc in &ctx.svc_ctxs { - *svc.project.write().await = Arc::downgrade(&ctx); + svc.set_project(Arc::downgrade(&ctx)).await; } ctx @@ -172,6 +172,7 @@ impl ProjectContextData { // Load the service let svc_ctx = context::service::ServiceContextData::from_path( Weak::new(), + &svc_ctxs_map, &workspace_path, &worker_path, ) @@ -218,6 +219,7 @@ impl ProjectContextData { // Load the service let svc_ctx = context::service::ServiceContextData::from_path( Weak::new(), + &svc_ctxs_map, workspace_path, &entry.path(), ) @@ -439,10 +441,11 @@ impl ProjectContextData { impl ProjectContextData { /// Origin used for building links to the Hub. pub fn origin_hub(&self) -> String { - self.ns().dns.hub_origin.clone().map_or_else( - || self.ns().dns.domain.main.clone(), - |x| format!("https://hub.{x}"), - ) + self.ns() + .dns + .hub_origin + .clone() + .unwrap_or_else(|| format!("https://hub.{}", self.ns().dns.domain.main.clone())) } pub fn domain_main(&self) -> String { self.ns().dns.domain.main.clone() @@ -532,65 +535,63 @@ pub struct S3Config { pub region: String, } -#[derive(Clone, Copy)] -pub enum S3Provider { - Minio, - Backblaze, - Aws, -} - impl ProjectContextData { - pub fn default_s3_provider(self: &Arc) -> Result<(S3Provider, config::ns::S3Provider)> { + pub fn default_s3_provider( + self: &Arc, + ) -> Result<(s3_util::Provider, config::ns::S3Provider)> { let providers = &self.ns().s3.providers; // Find the provider with the default flag set if let Some(p) = &providers.minio { if p.default { - return Ok((S3Provider::Minio, p.clone())); + return Ok((s3_util::Provider::Minio, p.clone())); } } if let Some(p) = &providers.backblaze { if p.default { - return Ok((S3Provider::Backblaze, p.clone())); + return Ok((s3_util::Provider::Backblaze, p.clone())); } } if let Some(p) = &providers.aws { if p.default { - return Ok((S3Provider::Aws, p.clone())); + return Ok((s3_util::Provider::Aws, p.clone())); } } // If nonoe have the default flag, return the first provider if let Some(p) = &providers.minio { - return Ok((S3Provider::Minio, p.clone())); + return Ok((s3_util::Provider::Minio, p.clone())); } else if let Some(p) = &providers.backblaze { - return Ok((S3Provider::Backblaze, p.clone())); + return Ok((s3_util::Provider::Backblaze, p.clone())); } else if let Some(p) = &providers.aws { - return Ok((S3Provider::Aws, p.clone())); + return Ok((s3_util::Provider::Aws, p.clone())); } bail!("no s3 provider configured") } /// Returns the appropriate S3 connection configuration for the provided S3 provider. - pub async fn s3_credentials(self: &Arc, provider: S3Provider) -> Result { + pub async fn s3_credentials( + self: &Arc, + provider: s3_util::Provider, + ) -> Result { match provider { - S3Provider::Minio => Ok(S3Credentials { + s3_util::Provider::Minio => Ok(S3Credentials { access_key_id: "root".into(), access_key_secret: self .read_secret(&["minio", "users", "root", "password"]) .await?, }), - S3Provider::Backblaze => { + s3_util::Provider::Backblaze => { let service_key = s3::fetch_service_key(&self, &["b2", "terraform"]).await?; Ok(S3Credentials { access_key_id: service_key.key_id, access_key_secret: service_key.key, }) } - S3Provider::Aws => { + s3_util::Provider::Aws => { let service_key = s3::fetch_service_key(&self, &["aws", "terraform"]).await?; Ok(S3Credentials { access_key_id: service_key.key_id, @@ -601,9 +602,9 @@ impl ProjectContextData { } /// Returns the appropriate S3 connection configuration for the provided S3 provider. - pub async fn s3_config(self: &Arc, provider: S3Provider) -> Result { + pub async fn s3_config(self: &Arc, provider: s3_util::Provider) -> Result { match provider { - S3Provider::Minio => { + s3_util::Provider::Minio => { let s3 = terraform::output::read_s3_minio(&*self).await; Ok(S3Config { endpoint_internal: (*s3.s3_endpoint_internal).clone(), @@ -611,7 +612,7 @@ impl ProjectContextData { region: (*s3.s3_region).clone(), }) } - S3Provider::Backblaze => { + s3_util::Provider::Backblaze => { let s3 = terraform::output::read_s3_backblaze(&*self).await; Ok(S3Config { endpoint_internal: (*s3.s3_endpoint_internal).clone(), @@ -619,7 +620,7 @@ impl ProjectContextData { region: (*s3.s3_region).clone(), }) } - S3Provider::Aws => { + s3_util::Provider::Aws => { let s3 = terraform::output::read_s3_aws(&*self).await; Ok(S3Config { endpoint_internal: (*s3.s3_endpoint_internal).clone(), diff --git a/lib/bolt/core/src/context/service.rs b/lib/bolt/core/src/context/service.rs index 95a8977a84..f7db5d75ee 100644 --- a/lib/bolt/core/src/context/service.rs +++ b/lib/bolt/core/src/context/service.rs @@ -1,12 +1,12 @@ +use anyhow::{bail, ensure, Context, Result}; +use async_recursion::async_recursion; +use s3_util::aws_sdk_s3; use std::{ collections::HashMap, hash::{Hash, Hasher}, path::{Path, PathBuf}, sync::{Arc, Weak}, }; - -use anyhow::{bail, ensure, Context, Result}; -use s3_util::aws_sdk_s3; use tempfile::NamedTempFile; use tokio::{fs, process::Command, sync::RwLock}; @@ -15,7 +15,7 @@ use crate::{ self, service::{RuntimeKind, ServiceKind}, }, - context::{self, BuildContext, ProjectContext, RunContext, S3Provider}, + context::{self, BuildContext, ProjectContext, RunContext}, dep::{self, cloudflare, s3}, utils, }; @@ -25,7 +25,9 @@ use super::BuildOptimization; pub type ServiceContext = Arc; pub struct ServiceContextData { - pub(in crate::context) project: RwLock>, + project: RwLock>, + /// If this is overriding a service from an additional root, then this will be specified. + overridden_service: Option, config: config::service::ServiceConfig, path: PathBuf, workspace_path: PathBuf, @@ -45,6 +47,18 @@ impl Hash for ServiceContextData { } impl ServiceContextData { + /// Sets the reference to the project once the project context is finished initiating. + #[async_recursion] + pub(in crate::context) async fn set_project( + &self, + project: Weak, + ) { + *self.project.write().await = project.clone(); + if let Some(overridden_service) = &self.overridden_service { + overridden_service.set_project(project.clone()).await; + } + } + pub async fn project(&self) -> ProjectContext { self.project .read() @@ -69,6 +83,7 @@ impl ServiceContextData { impl ServiceContextData { pub async fn from_path( project: Weak, + svc_ctxs_map: &HashMap, workspace_path: &Path, path: &Path, ) -> Option { @@ -92,8 +107,12 @@ impl ServiceContextData { Err(_) => None, }; + // Read overridden service + let overridden_service = svc_ctxs_map.get(&config.service.name).cloned(); + Some(Arc::new(ServiceContextData::new( project, + overridden_service, config, workspace_path, path, @@ -103,6 +122,7 @@ impl ServiceContextData { fn new( project: Weak, + overridden_service: Option, config: config::service::ServiceConfig, workspace_path: &Path, path: &Path, @@ -111,6 +131,7 @@ impl ServiceContextData { // Build context let ctx = ServiceContextData { project: RwLock::new(project), + overridden_service, config, path: path.to_owned(), workspace_path: workspace_path.to_owned(), @@ -324,7 +345,7 @@ impl ServiceContextData { impl ServiceContextData { pub fn enable_tokio_console(&self) -> bool { // TODO: This seems to have a significant performance impact - false + true } } @@ -332,16 +353,32 @@ impl ServiceContextData { #[derive(Debug, Clone)] pub enum ServiceBuildPlan { /// Build exists locally. - ExistingLocalBuild { output_path: PathBuf }, + ExistingLocalBuild { + /// Absolute path to the executable on the host system. + exec_path: PathBuf, + }, /// Build exists on S3 server. - ExistingUploadedBuild { build_key: String }, + ExistingUploadedBuild { + build_key: String, + + /// Path to the executable in the archive. + exec_path: String, + }, /// Build the service locally. - BuildLocally { output_path: PathBuf }, + BuildLocally { + /// Absolute path to the executable on the host system. + exec_path: PathBuf, + }, /// Build the service and upload to S3. - BuildAndUpload { build_key: String }, + BuildAndUpload { + build_key: String, + + /// Path to the executable in the archive. + exec_path: String, + }, /// Run a Docker container. Docker { image_tag: String }, @@ -369,36 +406,52 @@ impl ServiceContextData { .await; if force_build { - return Ok(ServiceBuildPlan::BuildLocally { output_path }); + return Ok(ServiceBuildPlan::BuildLocally { + exec_path: output_path, + }); } // Check if there's an existing build we can use let build_exists = tokio::fs::metadata(&output_path).await.is_ok(); if build_exists { - return Ok(ServiceBuildPlan::ExistingLocalBuild { output_path }); + return Ok(ServiceBuildPlan::ExistingLocalBuild { + exec_path: output_path, + }); } // Default to building - Ok(ServiceBuildPlan::BuildLocally { output_path }) + Ok(ServiceBuildPlan::BuildLocally { + exec_path: output_path, + }) } // Build and upload to S3 config::ns::ClusterKind::Distributed { .. } => { // Derive the build key let key = self.s3_build_key(build_context).await?; + let exec_path = self.cargo_name().expect("no cargo name").to_string(); if force_build { - return Ok(ServiceBuildPlan::BuildAndUpload { build_key: key }); + return Ok(ServiceBuildPlan::BuildAndUpload { + build_key: key, + exec_path, + }); } // Check if there's an existing build we can use let s3_client = project_ctx.s3_client_service_builds().await?; let build_exists = s3::check_exists_cached(&project_ctx, &s3_client, &key).await?; if build_exists { - return Ok(ServiceBuildPlan::ExistingUploadedBuild { build_key: key }); + return Ok(ServiceBuildPlan::ExistingUploadedBuild { + build_key: key, + exec_path, + }); } // Default to building - Ok(ServiceBuildPlan::BuildAndUpload { build_key: key }) + Ok(ServiceBuildPlan::BuildAndUpload { + build_key: key, + exec_path, + }) } } } @@ -406,6 +459,7 @@ impl ServiceContextData { // Dependencies impl ServiceContextData { + #[async_recursion] pub async fn dependencies(&self) -> Vec { let project = self.project().await; @@ -425,33 +479,7 @@ impl ServiceContextData { } } - // TODO: Add dev dependencies if building for tests - // Add operation dependencies from Cargo.toml - if let Some(cargo) = &self.cargo { - let svc_path = self.path(); - let svcs = cargo - .dependencies - .iter() - .filter_map(|(_, x)| { - if let config::service::CargoDependency::Path { path } = x { - Some(path) - } else { - None - } - }) - .filter_map(|path| { - let absolute_path = svc_path.join(path); - all_svcs - .iter() - .filter(|x| x.path() == absolute_path) - .next() - .cloned() - }); - - dep_ctxs.extend(svcs); - } - - // Check that these are services you can explicitly depend on + // Check that these are services you can explicitly depend on in the Service.toml for dep in &dep_ctxs { if !self.config().service.test_only && dep.config().service.test_only { panic!( @@ -473,6 +501,46 @@ impl ServiceContextData { } } + // TODO: Add dev dependencies if building for tests + // Add operation dependencies from Cargo.toml + // + // You cannot depend on these services from the Service.toml, only as a Cargo dependency + if let Some(cargo) = &self.cargo { + let svcs = cargo + .dependencies + .iter() + .filter_map(|(name, dep)| { + if let config::service::CargoDependency::Path { .. } = dep { + Some(name) + } else { + None + } + }) + .filter_map(|name| { + all_svcs + .iter() + .filter(|x| x.name() == *name) + .next() + .cloned() + }); + // TOOD: Use the path to find the service instead of the name. This is difficult with multiple roots. + // .filter_map(|path| { + // let absolute_path = svc_path.join(path); + // all_svcs + // .iter() + // .filter(|x| x.path() == absolute_path) + // .next() + // .cloned() + // }); + + dep_ctxs.extend(svcs); + } + + // Inherit dependencies from the service that was overridden. + if let Some(overriden_svc) = &self.overridden_service { + dep_ctxs.extend(overriden_svc.dependencies().await); + } + dep_ctxs } @@ -691,7 +759,7 @@ impl ServiceContextData { env.push(( "TOKIO_WORKER_THREADS".into(), match ns_service_config.resources.cpu { - config::ns::CpuResources::CpuCores(cores) => cores.max(2), + config::ns::CpuResources::CpuCores(cores) => cores.max(cores), config::ns::CpuResources::Cpu(_) => 2, } .to_string(), @@ -719,7 +787,7 @@ impl ServiceContextData { env.push(("TOKIO_CONSOLE_ENABLE".into(), "1".into())); env.push(( "TOKIO_CONSOLE_BIND".into(), - r#"0.0.0.0:{{env "NOMAD_PORT_tokio_console"}}"#.into(), + r#"0.0.0.0:${NOMAD_PORT_tokio-console}"#.into(), )); } @@ -873,7 +941,7 @@ impl ServiceContextData { // NATS config env.push(( "NATS_URL".into(), - // TODO: Add back passsing multiple NATS nodes for failover instead of using DNS resolution + // TODO: Add back passing multiple NATS nodes for failover instead of using DNS resolution access_service( &project_ctx, &mut tunnel_configs, @@ -991,44 +1059,28 @@ impl ServiceContextData { } // Add default provider - let default_provider_name = match project_ctx.default_s3_provider()? { - (S3Provider::Minio, _) => "MINIO", - (S3Provider::Backblaze, _) => "BACKBLAZE", - (S3Provider::Aws, _) => "AWS", - }; + let (default_provider, _) = project_ctx.default_s3_provider()?; env.push(( "S3_DEFAULT_PROVIDER".to_string(), - default_provider_name.to_string(), + default_provider.to_string(), )); // Add all configured providers let providers = &project_ctx.ns().s3.providers; if providers.minio.is_some() { - add_s3_env( - &project_ctx, - &mut env, - &s3_dep, - context::project::S3Provider::Minio, - ) - .await?; + add_s3_env(&project_ctx, &mut env, &s3_dep, s3_util::Provider::Minio).await?; } if providers.backblaze.is_some() { add_s3_env( &project_ctx, &mut env, &s3_dep, - context::project::S3Provider::Backblaze, + s3_util::Provider::Backblaze, ) .await?; } if providers.aws.is_some() { - add_s3_env( - &project_ctx, - &mut env, - &s3_dep, - context::project::S3Provider::Aws, - ) - .await?; + add_s3_env(&project_ctx, &mut env, &s3_dep, s3_util::Provider::Aws).await?; } } @@ -1219,7 +1271,7 @@ impl ServiceContextData { } ); - let service = project_ctx + let mut service = project_ctx .ns() .services .get(&self.name()) @@ -1235,7 +1287,7 @@ impl ServiceContextData { }, }, config::ns::ClusterKind::Distributed { .. } => config::ns::Service { - count: if is_singleton { 1 } else { 2 }, + count: 2, resources: config::ns::ServiceResources { cpu: config::ns::CpuResources::Cpu(250), memory: 256, @@ -1244,8 +1296,9 @@ impl ServiceContextData { }, }); + // Force single count if singleton if is_singleton { - assert_eq!(service.count, 1) + service.count = 1; } service @@ -1256,13 +1309,9 @@ async fn add_s3_env( project_ctx: &ProjectContext, env: &mut Vec<(String, String)>, s3_dep: &Arc, - provider: S3Provider, + provider: s3_util::Provider, ) -> Result<()> { - let provider_name = match provider { - S3Provider::Minio => "MINIO", - S3Provider::Backblaze => "BACKBLAZE", - S3Provider::Aws => "AWS", - }; + let provider_name = provider.to_string(); let s3_dep_name = s3_dep.name_screaming_snake(); let s3_config = project_ctx.s3_config(provider).await?; let s3_creds = project_ctx.s3_credentials(provider).await?; diff --git a/lib/bolt/core/src/dep/cargo/cli.rs b/lib/bolt/core/src/dep/cargo/cli.rs index 638147008e..8171f3f9b4 100644 --- a/lib/bolt/core/src/dep/cargo/cli.rs +++ b/lib/bolt/core/src/dep/cargo/cli.rs @@ -61,12 +61,10 @@ pub async fn build<'a, T: AsRef>(ctx: &ProjectContext, opts: BuildOpts<'a, .collect::>() .join(" "); - // TODO: Not sure why the .cargo/config.toml isn't working with nested projects, have to hardcode - // the target dir indoc::formatdoc!( " if [ $? -eq 0 ]; then - (cd {path} && cargo build {jobs_flag} {format_flag} {release_flag} {bin_flags} --target-dir $TARGET_DIR) + (cd {path} && cargo build {jobs_flag} {format_flag} {release_flag} {bin_flags}) fi " ) @@ -77,7 +75,9 @@ pub async fn build<'a, T: AsRef>(ctx: &ProjectContext, opts: BuildOpts<'a, // Generate build script let build_script = indoc::formatdoc!( r#" - TARGET_DIR=$(readlink -f ./target) + # TODO: Not sure why the .cargo/config.toml isn't working with nested projects, have to hardcode + # the target dir + export CARGO_TARGET_DIR={target} # Used for Tokio Console. See https://github.com/tokio-rs/console#using-it export RUSTFLAGS="--cfg tokio_unstable" # Used for debugging @@ -87,6 +87,7 @@ pub async fn build<'a, T: AsRef>(ctx: &ProjectContext, opts: BuildOpts<'a, EXIT_CODE=$? "#, + target = ctx.path().join("target").display(), ); // Execute build command @@ -116,7 +117,7 @@ pub async fn build<'a, T: AsRef>(ctx: &ProjectContext, opts: BuildOpts<'a, cmd.arg("--rm") .arg("--interactive") .arg("--tty") - .arg("clux/muslrust:1.65.0-stable"); + .arg("clux/muslrust:1.71.1-stable"); cmd.arg("sh").arg("-c").arg(indoc::formatdoc!( r#" # HACK: Link musl-g++ diff --git a/lib/bolt/core/src/dep/nomad/gen.rs b/lib/bolt/core/src/dep/nomad/gen.rs index b7b52ace76..bb48a3bf2c 100644 --- a/lib/bolt/core/src/dep/nomad/gen.rs +++ b/lib/bolt/core/src/dep/nomad/gen.rs @@ -25,11 +25,13 @@ pub enum ExecServiceDriver { }, UploadedBinaryArtifact { artifact_key: String, + /// Path to the executable within the archive. + exec_path: String, args: Vec, }, LocalBinaryArtifact { - /// Path relative to the project root. - path: PathBuf, + /// Path to the executable relative to the project root. + exec_path: PathBuf, args: Vec, }, } @@ -183,32 +185,32 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { name: Some("svc".into()), // TODO: Add autoscaling count: Some(ns_service_config.count as i64), - update: if enable_update { - Some( - // Wait for services to be healthy before progressing deploy - if has_health { - Update { - max_parallel: Some(1), - health_check: Some("checks".to_owned()), - min_healthy_time: Some(chrono::secs(10)), - healthy_deadline: Some(chrono::min(5)), - progress_deadline: Some(chrono::min(10)), - auto_revert: Some(true), - canary: Some(0), - stagger: Some(chrono::secs(30)), - ..Default::default() - } - } else { - // Just monitor the task status - Update { - health_check: Some("task_states".into()), - ..Default::default() - } - }, - ) - } else { - None - }, + // update: if enable_update { + // Some( + // // Wait for services to be healthy before progressing deploy + // if has_health { + // Update { + // max_parallel: Some(1), + // health_check: Some("checks".to_owned()), + // min_healthy_time: Some(chrono::secs(10)), + // healthy_deadline: Some(chrono::min(5)), + // progress_deadline: Some(chrono::min(10)), + // auto_revert: Some(true), + // canary: Some(0), + // stagger: Some(chrono::secs(30)), + // ..Default::default() + // } + // } else { + // // Just monitor the task status + // Update { + // health_check: Some("task_states".into()), + // ..Default::default() + // } + // }, + // ) + // } else { + // None + // }, reschedule_policy: Some(ReschedulePolicy { delay: Some(chrono::secs(30)), delay_function: Some("constant".into()), @@ -239,7 +241,7 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { }]), ephemeral_disk: Some(json!({ // Prevent exhausting all storage resources on the local machine. We don't - // accept anything over 512 MB since our dev machiens are limited in space. + // accept anything over 512 MB since our dev machines are limited in space. "SizeMB": ns_service_config.resources.ephemeral_disk, })), services: Some({ @@ -268,15 +270,15 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { let mut checks = Vec::new(); // Require a healthy health check before booting - let interval = chrono::secs(15); - let on_update = "require_healthy"; + let interval = chrono::secs(5); + let on_update = "ignore"; checks.push(Check { name: Some("Health Server Liveness".into()), check_type: Some("http".into()), port_label: Some("health".into()), path: Some("/health/liveness".into()), - interval: Some(chrono::secs(15)), + interval: Some(chrono::secs(5)), timeout: Some(chrono::secs(5)), on_update: Some(on_update.into()), // This indicates the runtime might @@ -286,13 +288,20 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { // // Ignore in staging since we have // `on_update` set to `ignore`. - check_restart: Some(CheckRestart { - limit: Some(3), - ..Default::default() - }), + // check_restart: Some(CheckRestart { + // limit: Some(3), + // ..Default::default() + // }), ..Default::default() }); + checks.push(build_conn_check( + "Cockroach", + "/health/crdb/db-user", + interval, + on_update, + )); + checks.push(build_conn_check( "Nats Connection", "/health/nats", @@ -425,19 +434,19 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { "logging": nomad_loki_plugin_config(&project_ctx, &svc_ctx).await.unwrap(), }) } - ExecServiceDriver::LocalBinaryArtifact { path, args } => { + ExecServiceDriver::LocalBinaryArtifact { exec_path, args } => { json!({ "image": "alpine:3.18", "args": args, - "command": Path::new("/var/rivet/backend").join(path), + "command": Path::new("/var/rivet/backend").join(exec_path), "auth": nomad_docker_io_auth(&project_ctx).await.unwrap(), "logging": nomad_loki_plugin_config(&project_ctx, &svc_ctx).await.unwrap(), }) } - ExecServiceDriver::UploadedBinaryArtifact { args, .. } => { + ExecServiceDriver::UploadedBinaryArtifact { exec_path, args, .. } => { json!({ "image": "alpine:3.18", - "command": format!("${{NOMAD_TASK_DIR}}/build/{}", svc_ctx.name()), + "command": format!("${{NOMAD_TASK_DIR}}/build/{exec_path}"), "args": args, "auth": nomad_docker_io_auth(&project_ctx).await.unwrap(), "logging": nomad_loki_plugin_config(&project_ctx, &svc_ctx).await.unwrap(), @@ -469,15 +478,28 @@ pub async fn gen_svc(region_id: &str, exec_ctx: &ExecServiceContext) -> Job { let s3_config = project_ctx.s3_config(default_provider).await.unwrap(); let s3_creds = project_ctx.s3_credentials(default_provider).await.unwrap(); + // The getter source is not a standard URL, so we have to hardcode it + // ourselves. + // + // Using the region in the domain causes it to throw an invalid URL error. + let getter_source = match default_provider { + s3_util::Provider::Aws => { + format!("s3::https://s3.amazonaws.com/{bucket}/{artifact_key}") + } + s3_util::Provider::Backblaze => { + format!( + "s3::{endpoint}/{bucket}/{key}", + endpoint = s3_config.endpoint_external, + key = urlencoding::encode(artifact_key), + ) + } + _ => todo!(), + }; + Some(json!([ { "GetterMode": "dir", - "GetterSource": format!( - "s3::{endpoint}/{bucket}/{key}", - endpoint = s3_config.endpoint_internal, - bucket = bucket, - key = urlencoding::encode(artifact_key), - ), + "GetterSource": getter_source, "GetterOptions": { "region": s3_config.region, "aws_access_key_id": s3_creds.access_key_id, diff --git a/lib/bolt/core/src/dep/salt/cli.rs b/lib/bolt/core/src/dep/salt/cli.rs index 2c2cc8fa55..b11518fa78 100644 --- a/lib/bolt/core/src/dep/salt/cli.rs +++ b/lib/bolt/core/src/dep/salt/cli.rs @@ -31,7 +31,19 @@ pub async fn apply( let mut event = utils::telemetry::build_event(ctx, "bolt_salt_apply").await?; event.insert_prop("filter", filter)?; event.insert_prop("sls", &opts.sls)?; - utils::telemetry::capture_event(ctx, event).await?; + utils::telemetry::capture_event(ctx, event)?; + + // Install rsync + if let config::ns::ClusterKind::Distributed { .. } = ctx.ns().cluster.kind { + eprintln!(); + rivet_term::status::progress("Installing rsync on Salt Master", ""); + tasks::ssh::pool( + &ctx, + "salt_master", + Some("if ! which rsync; then apt update -y && apt install -y rsync; fi"), + ) + .await?; + } // Write Salt configs eprintln!(); @@ -59,19 +71,22 @@ pub async fn apply( } config::ns::ClusterKind::Distributed { .. } => { tokio::try_join!( - // /srv/salt - async { rsync_dir(ctx, &ctx.salt_path().join("salt"), "/srv/salt").await }, - // /srv/pillar - async { rsync_dir(ctx, &ctx.salt_path().join("pillar"), "/srv/pillar").await }, - // /srv/rivet-nix async { + // /srv/salt + rsync_dir(ctx, &ctx.salt_path().join("salt"), "/srv/salt").await?; + + // /srv/salt/nix/files/source rsync_dir( ctx, - &ctx.salt_path().join("pillar"), + &ctx.path().join("infra").join("nix"), "/srv/salt/nix/files/source", ) - .await + .await?; + + Ok(()) }, + // /srv/pillar + async { rsync_dir(ctx, &ctx.salt_path().join("pillar"), "/srv/pillar").await }, // /srv/salt-context async { let tmp_dir = tempfile::TempDir::new()?; diff --git a/lib/bolt/core/src/dep/salt/config.rs b/lib/bolt/core/src/dep/salt/config.rs index bda33f0707..c4d5455835 100644 --- a/lib/bolt/core/src/dep/salt/config.rs +++ b/lib/bolt/core/src/dep/salt/config.rs @@ -3,7 +3,7 @@ use serde_json::{json, Value}; use crate::{ config::{self, ns::LoggingProvider, service::RuntimeKind}, - context::{ProjectContext, S3Provider}, + context::ProjectContext, dep, }; @@ -49,7 +49,11 @@ pub async fn build(ctx: &ProjectContext, opts: &BuildOpts) -> Result { vars["cloudflare"] = cloudflare(ctx)?; - vars["s3"] = s3(ctx, opts.skip_s3).await?; + vars["s3"] = if opts.skip_s3 { + json!({}) + } else { + s3(ctx).await? + }; vars["logging"] = logging(ctx)?; @@ -60,23 +64,9 @@ pub async fn build(ctx: &ProjectContext, opts: &BuildOpts) -> Result { Ok(vars) } -async fn s3(ctx: &ProjectContext, skip: bool) -> Result { +async fn s3(ctx: &ProjectContext) -> Result { let mut res = serde_json::Map::with_capacity(1); - if skip { - // Provide filler values so the pillars can still render - res.insert( - "default".to_string(), - json!({ - "endpoint_internal": "", - "endpoint_external": "", - "region": "", - }), - ); - - return Ok(res.into()); - } - let (default_provider, _) = ctx.default_s3_provider()?; let default_s3_config = ctx.s3_config(default_provider).await?; res.insert( @@ -90,7 +80,7 @@ async fn s3(ctx: &ProjectContext, skip: bool) -> Result { let providers = &ctx.ns().s3.providers; if providers.minio.is_some() { - let s3_config = ctx.s3_config(S3Provider::Minio).await?; + let s3_config = ctx.s3_config(s3_util::Provider::Minio).await?; res.insert( "minio".to_string(), json!({ @@ -101,7 +91,7 @@ async fn s3(ctx: &ProjectContext, skip: bool) -> Result { ); } if providers.backblaze.is_some() { - let s3_config = ctx.s3_config(S3Provider::Backblaze).await?; + let s3_config = ctx.s3_config(s3_util::Provider::Backblaze).await?; res.insert( "backblaze".to_string(), json!({ @@ -112,7 +102,7 @@ async fn s3(ctx: &ProjectContext, skip: bool) -> Result { ); } if providers.aws.is_some() { - let s3_config = ctx.s3_config(S3Provider::Aws).await?; + let s3_config = ctx.s3_config(s3_util::Provider::Aws).await?; res.insert( "aws".to_string(), json!({ diff --git a/lib/bolt/core/src/dep/salt/secrets.rs b/lib/bolt/core/src/dep/salt/secrets.rs index 89ae47c0a6..bd13b2c3b1 100644 --- a/lib/bolt/core/src/dep/salt/secrets.rs +++ b/lib/bolt/core/src/dep/salt/secrets.rs @@ -1,7 +1,7 @@ use anyhow::*; use serde_json::{json, Value}; -use crate::context::{ProjectContext, S3Provider}; +use crate::context::ProjectContext; /// Generates a config that will be exposed to Salt. pub async fn build_secrets(ctx: &ProjectContext) -> Result { @@ -15,6 +15,16 @@ pub async fn build_secrets(ctx: &ProjectContext) -> Result { secrets["s3"] = s3(ctx).await?; + if ctx.ns().s3.providers.minio.is_some() { + secrets["minio"] = json!({ + "users": { + "root": { + "password": ctx.read_secret(&["minio", "users", "root", "password"]).await?, + }, + }, + }); + } + secrets["clickhouse"] = json!({ "users": { "bolt": { @@ -29,14 +39,6 @@ pub async fn build_secrets(ctx: &ProjectContext) -> Result { }, }); - secrets["minio"] = json!({ - "users": { - "root": { - "password": ctx.read_secret(&["minio", "users", "root", "password"]).await?, - }, - }, - }); - if let (Some(client_id), Some(client_secret)) = ( ctx.read_secret_opt(&["cloudflare", "access", "proxy", "client_id"]) .await?, @@ -71,7 +73,7 @@ async fn s3(ctx: &ProjectContext) -> Result { let providers = &ctx.ns().s3.providers; if providers.minio.is_some() { - let s3_creds = ctx.s3_credentials(S3Provider::Minio).await?; + let s3_creds = ctx.s3_credentials(s3_util::Provider::Minio).await?; res.insert( "minio".to_string(), json!({ @@ -81,7 +83,7 @@ async fn s3(ctx: &ProjectContext) -> Result { ); } if providers.backblaze.is_some() { - let s3_creds = ctx.s3_credentials(S3Provider::Backblaze).await?; + let s3_creds = ctx.s3_credentials(s3_util::Provider::Backblaze).await?; res.insert( "backblaze".to_string(), json!({ @@ -91,7 +93,7 @@ async fn s3(ctx: &ProjectContext) -> Result { ); } if providers.aws.is_some() { - let s3_creds = ctx.s3_credentials(S3Provider::Aws).await?; + let s3_creds = ctx.s3_credentials(s3_util::Provider::Aws).await?; res.insert( "aws".to_string(), json!({ diff --git a/lib/bolt/core/src/dep/terraform/.nebula_firewall_rules.rs.swp b/lib/bolt/core/src/dep/terraform/.nebula_firewall_rules.rs.swp new file mode 100644 index 0000000000..13b7ea1f9c Binary files /dev/null and b/lib/bolt/core/src/dep/terraform/.nebula_firewall_rules.rs.swp differ diff --git a/lib/bolt/core/src/dep/terraform/.pools.rs.swp b/lib/bolt/core/src/dep/terraform/.pools.rs.swp new file mode 100644 index 0000000000..e2b3954f60 Binary files /dev/null and b/lib/bolt/core/src/dep/terraform/.pools.rs.swp differ diff --git a/lib/bolt/core/src/dep/terraform/cli.rs b/lib/bolt/core/src/dep/terraform/cli.rs index 44b03e35a7..0534aa6b1f 100644 --- a/lib/bolt/core/src/dep/terraform/cli.rs +++ b/lib/bolt/core/src/dep/terraform/cli.rs @@ -100,7 +100,7 @@ pub async fn apply( ) -> Result<()> { let mut event = utils::telemetry::build_event(ctx, "bolt_terraform_apply").await?; event.insert_prop("plan_id", plan_id)?; - utils::telemetry::capture_event(ctx, event).await?; + utils::telemetry::capture_event(ctx, event)?; let mut cmd = build_command(ctx, plan_id).await; cmd.arg("apply") @@ -117,7 +117,7 @@ pub async fn apply( pub async fn destroy(ctx: &ProjectContext, plan_id: &str, varfile_path: &Path) -> Result<()> { let mut event = utils::telemetry::build_event(ctx, "bolt_terraform_destroy").await?; event.insert_prop("plan_id", plan_id)?; - utils::telemetry::capture_event(ctx, event).await?; + utils::telemetry::capture_event(ctx, event)?; let mut cmd = build_command(&ctx, plan_id).await; cmd.arg("destroy") diff --git a/lib/bolt/core/src/dep/terraform/gen.rs b/lib/bolt/core/src/dep/terraform/gen.rs index 69e9b183c0..f82d794d25 100644 --- a/lib/bolt/core/src/dep/terraform/gen.rs +++ b/lib/bolt/core/src/dep/terraform/gen.rs @@ -389,6 +389,25 @@ async fn vars(ctx: &ProjectContext) { "s3_persistent_access_key_secret".into(), json!(credentials.access_key_secret), ); + + // Build providers list + let ns_s3_providers = &ctx.ns().s3.providers; + let mut s3_providers = Vec::with_capacity(1); + + if ns_s3_providers.backblaze.is_some() { + s3_providers.push("backblaze"); + } + if ns_s3_providers.minio.is_some() { + s3_providers.push("minio"); + } + if ns_s3_providers.aws.is_some() { + s3_providers.push("aws"); + } + + vars.insert( + "s3_providers".into(), + Into::::into(s3_providers), + ); } // Media presets diff --git a/lib/bolt/core/src/dep/terraform/nebula_firewall_rules.rs b/lib/bolt/core/src/dep/terraform/nebula_firewall_rules.rs index 7343a1feeb..fa3be47e6b 100644 --- a/lib/bolt/core/src/dep/terraform/nebula_firewall_rules.rs +++ b/lib/bolt/core/src/dep/terraform/nebula_firewall_rules.rs @@ -119,6 +119,7 @@ pub fn traffic_server() -> Vec { vec![ Rule::group("pool:svc", RuleProtocol::TCP, "9300"), Rule::group("pool:ing-px", RuleProtocol::TCP, "9300"), + Rule::group("pool:job", RuleProtocol::TCP, "9300"), ] } diff --git a/lib/bolt/core/src/dep/terraform/pools.rs b/lib/bolt/core/src/dep/terraform/pools.rs index f6c563938f..60e4d07870 100644 --- a/lib/bolt/core/src/dep/terraform/pools.rs +++ b/lib/bolt/core/src/dep/terraform/pools.rs @@ -60,6 +60,9 @@ pub enum PoolLocalMode { /// Run this only when developing locally. LocalOnly, + /// Run this locally and remotely when developing. + LocalAndRemote, + /// Treat the pool normally. Keep, } @@ -186,18 +189,8 @@ pub async fn build_pools(ctx: &ProjectContext) -> Result> ); let mut svc_roles = vec!["docker", "nomad-client", "consul-client"]; - // Add logging roles - if ctx.ns().logging.is_some() - && ctx - .read_secret_opt(&["cloudflare", "access", "proxy", "client_id"]) - .await? - .is_some() - && ctx - .read_secret_opt(&["cloudflare", "access", "proxy", "client_secret"]) - .await? - .is_some() - { - svc_roles.extend(["traefik-cloudflare-proxy", "docker-plugin-loki"]); + if ctx.ns().logging.is_some() { + svc_roles.extend(["traefik", "cloudflare-proxy", "docker-plugin-loki"]); } pools.insert( "svc".into(), @@ -250,7 +243,7 @@ pub async fn build_pools(ctx: &ProjectContext) -> Result> .vpc(true) // TODO: We need a new PoolLocalMode for running locally and being able to run remote // nodes - .local_mode(PoolLocalMode::Locally) + .local_mode(PoolLocalMode::LocalAndRemote) .volumes(hashmap! { "ats".into() => PoolVolume {}, }) @@ -469,7 +462,7 @@ pub async fn build_pools(ctx: &ProjectContext) -> Result> pools.insert( "ing-px".into(), PoolBuilder::default() - .roles(vec!["traefik", "consul-client"]) + .roles(vec!["traefik", "ingress-proxy", "consul-client"]) .vpc(true) .local_mode(PoolLocalMode::Locally) .tunnels(hashmap! { @@ -549,7 +542,7 @@ pub async fn build_pools(ctx: &ProjectContext) -> Result> pools.insert( "ing-job".into(), PoolBuilder::default() - .roles(vec!["traefik"]) + .roles(vec!["traefik", "ingress-proxy"]) .vpc(false) .local_mode(PoolLocalMode::Keep) .tunnels(hashmap! { @@ -659,7 +652,7 @@ pub async fn build_pools(ctx: &ProjectContext) -> Result> }, FirewallRule { label: "nomad-dynamic-udp".into(), - ports: "20000-25999".into(), + ports: "20000-31999".into(), protocol: "udp".into(), inbound_ipv4_cidr: vec!["0.0.0.0/0".into()], inbound_ipv6_cidr: vec!["::/0".into()], @@ -715,7 +708,10 @@ fn filter_pools( new_pools.extend( pools .iter() - .filter(|(_, x)| x.local_mode == PoolLocalMode::Keep) + .filter(|(_, x)| { + x.local_mode == PoolLocalMode::Keep + || x.local_mode == PoolLocalMode::LocalAndRemote + }) .map(|(k, v)| (k.clone(), v.clone())), ); @@ -725,6 +721,7 @@ fn filter_pools( .filter(|(_, x)| { x.local_mode == PoolLocalMode::Locally || x.local_mode == PoolLocalMode::LocalOnly + || x.local_mode == PoolLocalMode::LocalAndRemote }) .map(|(_, x)| x) .collect::>(); diff --git a/lib/bolt/core/src/dep/terraform/remote_states.rs b/lib/bolt/core/src/dep/terraform/remote_states.rs index a5901d1f24..ded39cde1c 100644 --- a/lib/bolt/core/src/dep/terraform/remote_states.rs +++ b/lib/bolt/core/src/dep/terraform/remote_states.rs @@ -2,7 +2,7 @@ use derive_builder::Builder; use maplit::hashmap; use std::collections::HashMap; -use crate::context::{ProjectContext, S3Provider}; +use crate::context::ProjectContext; /// Defines the dependency graph for the Terraform plans. /// @@ -16,9 +16,9 @@ pub fn dependency_graph(ctx: &ProjectContext) -> HashMap<&'static str, Vec { let (default_s3_provider, _) = ctx.default_s3_provider().unwrap(); let provider_plan_id = match default_s3_provider { - S3Provider::Minio => "s3_minio", - S3Provider::Backblaze => "s3_backblaze", - S3Provider::Aws => "s3_aws", + s3_util::Provider::Minio => "s3_minio", + s3_util::Provider::Backblaze => "s3_backblaze", + s3_util::Provider::Aws => "s3_aws", }; vec![RemoteStateBuilder::default() diff --git a/lib/bolt/core/src/tasks/api.rs b/lib/bolt/core/src/tasks/api.rs index 772ade07ee..812586cfae 100644 --- a/lib/bolt/core/src/tasks/api.rs +++ b/lib/bolt/core/src/tasks/api.rs @@ -34,7 +34,7 @@ pub async fn convert_team(project_ctx: &ProjectContext, team_id: String) -> Resu bail!( "failed to convert team ({}):\n{:#?}", response.status().as_u16(), - response.json().await? + response.json::().await? ); } diff --git a/lib/bolt/core/src/tasks/check.rs b/lib/bolt/core/src/tasks/check.rs index 11a72c0480..1aa9ec00e7 100644 --- a/lib/bolt/core/src/tasks/check.rs +++ b/lib/bolt/core/src/tasks/check.rs @@ -122,6 +122,7 @@ async fn check_svcs( let mut cmd = Command::new("cargo"); cmd.current_dir(path); cmd.env("RUSTFLAGS", "--cfg tokio_unstable"); + cmd.env("CARGO_TARGET_DIR", ctx.path().join("target")); cmd.arg("clippy"); // Check tests, which will also check the main module. Using diff --git a/lib/bolt/core/src/tasks/config/generate.rs b/lib/bolt/core/src/tasks/config/generate.rs index ce98108084..2d015e1a93 100644 --- a/lib/bolt/core/src/tasks/config/generate.rs +++ b/lib/bolt/core/src/tasks/config/generate.rs @@ -431,7 +431,7 @@ pub async fn generate(project_path: &Path, ns_id: &str) -> Result<()> { let mut event = utils::telemetry::build_event(&ctx, "bolt_config_generate").await?; event.insert_prop("ns_id", ns_id)?; - utils::telemetry::capture_event(&ctx, event).await?; + utils::telemetry::capture_event(&ctx, event)?; eprintln!(); rivet_term::status::success( diff --git a/lib/bolt/core/src/tasks/infra/migrate/break_infra_terraform_monolith.rs b/lib/bolt/core/src/tasks/infra/migrate/break_infra_terraform_monolith.rs index 6c98b9444b..4f40a56016 100644 --- a/lib/bolt/core/src/tasks/infra/migrate/break_infra_terraform_monolith.rs +++ b/lib/bolt/core/src/tasks/infra/migrate/break_infra_terraform_monolith.rs @@ -21,10 +21,10 @@ pub async fn run(ctx: &ProjectContext) -> Result<()> { crate::tasks::gen::generate_project(&ctx).await; migrate_nebula(ctx).await?; migrate_master_cluster(ctx).await?; - // migrate_pools(ctx).await?; - // migrate_dns(ctx).await?; - // migrate_cloudflare_workers(ctx).await?; - // migrate_cloudflare_tunnels(ctx).await?; + migrate_pools(ctx).await?; + migrate_dns(ctx).await?; + migrate_cloudflare_workers(ctx).await?; + migrate_cloudflare_tunnels(ctx).await?; migrate_s3(ctx).await?; Ok(()) } @@ -429,14 +429,14 @@ async fn migrate_dns(ctx: &ProjectContext) -> Result<()> { ) .await?; - copy_resource( - ctx, - &src_infra, - (None, "managed", "cloudflare_certificate_pack", "rivet_game"), - &mut dst, - (None, "managed", "cloudflare_certificate_pack", "rivet_game"), - ) - .await?; + // copy_resource( + // ctx, + // &src_infra, + // (None, "managed", "cloudflare_certificate_pack", "rivet_game"), + // &mut dst, + // (None, "managed", "cloudflare_certificate_pack", "rivet_game"), + // ) + // .await?; copy_resource( ctx, diff --git a/lib/bolt/core/src/tasks/infra/mod.rs b/lib/bolt/core/src/tasks/infra/mod.rs index 33df4a785d..bb9f529988 100644 --- a/lib/bolt/core/src/tasks/infra/mod.rs +++ b/lib/bolt/core/src/tasks/infra/mod.rs @@ -16,7 +16,13 @@ pub struct ExecutePlanOpts { } #[derive(Debug, Clone)] -pub enum PlanStep { +pub struct PlanStep { + pub name_id: &'static str, + pub kind: PlanStepKind, +} + +#[derive(Debug, Clone)] +pub enum PlanStepKind { Terraform { plan_id: String, @@ -38,10 +44,10 @@ pub enum PlanStep { Up, } -impl PlanStep { +impl PlanStepKind { async fn execute(&self, ctx: ProjectContext, opts: &ExecutePlanOpts) -> Result<()> { match self { - PlanStep::Terraform { plan_id, .. } => { + PlanStepKind::Terraform { plan_id, .. } => { let varfile_path = ctx.gen_tf_env_path(); let mut cmd = terraform::cli::build_command(&ctx, plan_id).await; @@ -55,7 +61,7 @@ impl PlanStep { terraform::output::clear_cache(&ctx, &plan_id).await; } - PlanStep::Salt { + PlanStepKind::Salt { filter, sls, config_opts, @@ -70,10 +76,10 @@ impl PlanStep { salt::cli::apply_all(&ctx, &apply_opts, config_opts).await?; } } - PlanStep::Migrate => { + PlanStepKind::Migrate => { tasks::migrate::up_all(&ctx).await?; } - PlanStep::Up => { + PlanStepKind::Up => { tasks::up::up_all( &ctx, tasks::up::UpOpts { @@ -90,7 +96,7 @@ impl PlanStep { async fn destroy(&self, ctx: ProjectContext, opts: &ExecutePlanOpts) -> Result<()> { match self { - PlanStep::Terraform { + PlanStepKind::Terraform { plan_id, needs_destroy, } => { @@ -111,7 +117,7 @@ impl PlanStep { terraform::output::clear_cache(&ctx, &plan_id).await; } - PlanStep::Salt { .. } | PlanStep::Migrate | PlanStep::Up => { + PlanStepKind::Salt { .. } | PlanStepKind::Migrate | PlanStepKind::Up => { // Do nothing } } @@ -120,70 +126,97 @@ impl PlanStep { } } -pub fn build_plan(ctx: &ProjectContext) -> Result> { +pub fn build_plan(ctx: &ProjectContext, start_at: Option) -> Result> { let mut plan = Vec::new(); // TLS - plan.push(PlanStep::Terraform { - plan_id: "tls".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-tls", + kind: PlanStepKind::Terraform { + plan_id: "tls".into(), + needs_destroy: true, + }, }); // Nebula - plan.push(PlanStep::Terraform { - plan_id: "nebula".into(), - needs_destroy: false, + plan.push(PlanStep { + name_id: "tf-nebula", + kind: PlanStepKind::Terraform { + plan_id: "nebula".into(), + needs_destroy: false, + }, }); // Master match ctx.ns().cluster.kind { ClusterKind::SingleNode { .. } => { - plan.push(PlanStep::Terraform { - plan_id: "master_local".into(), - needs_destroy: false, + plan.push(PlanStep { + name_id: "tf-master-local", + kind: PlanStepKind::Terraform { + plan_id: "master_local".into(), + needs_destroy: false, + }, }); } ClusterKind::Distributed { .. } => { - plan.push(PlanStep::Terraform { - plan_id: "master_cluster".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-master-cluster", + kind: PlanStepKind::Terraform { + plan_id: "master_cluster".into(), + needs_destroy: true, + }, }); } } // Pools - plan.push(PlanStep::Terraform { - plan_id: "pools".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-pools", + kind: PlanStepKind::Terraform { + plan_id: "pools".into(), + needs_destroy: true, + }, }); // DNS - plan.push(PlanStep::Terraform { - plan_id: "dns".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-dns", + kind: PlanStepKind::Terraform { + plan_id: "dns".into(), + needs_destroy: true, + }, }); // Cloudflare - plan.push(PlanStep::Terraform { - plan_id: "cloudflare_workers".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-cf-workers", + kind: PlanStepKind::Terraform { + plan_id: "cloudflare_workers".into(), + needs_destroy: true, + }, }); if let config::ns::DnsProvider::Cloudflare { access: Some(_), .. } = ctx.ns().dns.provider { - plan.push(PlanStep::Terraform { - plan_id: "cloudflare_tunnels".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-cf-tunnels", + kind: PlanStepKind::Terraform { + plan_id: "cloudflare_tunnels".into(), + needs_destroy: true, + }, }); } // Grafana if ctx.ns().grafana.is_some() { - plan.push(PlanStep::Terraform { - plan_id: "grafana".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-grafana", + kind: PlanStepKind::Terraform { + plan_id: "grafana".into(), + needs_destroy: true, + }, }); } @@ -191,55 +224,89 @@ pub fn build_plan(ctx: &ProjectContext) -> Result> { let s3_providers = &ctx.ns().s3.providers; if s3_providers.minio.is_some() { // Install Minio for s3_minio Terraform plan - plan.push(PlanStep::Salt { - filter: Some("G@roles:minio".into()), - sls: None, - config_opts: salt::config::BuildOpts { skip_s3: true }, + plan.push(PlanStep { + name_id: "salt-minio", + kind: PlanStepKind::Salt { + filter: Some("G@roles:minio".into()), + sls: None, + config_opts: salt::config::BuildOpts { skip_s3: true }, + }, }); - plan.push(PlanStep::Terraform { - plan_id: "s3_minio".into(), - needs_destroy: false, + plan.push(PlanStep { + name_id: "tf-s3-minio", + kind: PlanStepKind::Terraform { + plan_id: "s3_minio".into(), + needs_destroy: false, + }, }); } if s3_providers.backblaze.is_some() { - plan.push(PlanStep::Terraform { - plan_id: "s3_backblaze".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-s3-backblaze", + kind: PlanStepKind::Terraform { + plan_id: "s3_backblaze".into(), + needs_destroy: true, + }, }); } if s3_providers.aws.is_some() { - plan.push(PlanStep::Terraform { - plan_id: "s3_aws".into(), - needs_destroy: true, + plan.push(PlanStep { + name_id: "tf-s3-aws", + kind: PlanStepKind::Terraform { + plan_id: "s3_aws".into(), + needs_destroy: true, + }, }); } // Apply the rest of the Salt configs - plan.push(PlanStep::Salt { - filter: None, - sls: None, - config_opts: Default::default(), + plan.push(PlanStep { + name_id: "salt", + kind: PlanStepKind::Salt { + filter: None, + sls: None, + config_opts: Default::default(), + }, }); - plan.push(PlanStep::Terraform { - plan_id: "nomad".into(), - needs_destroy: false, + plan.push(PlanStep { + name_id: "tf-nomad", + kind: PlanStepKind::Terraform { + plan_id: "nomad".into(), + needs_destroy: false, + }, }); - plan.push(PlanStep::Migrate); + plan.push(PlanStep { + name_id: "migrate", + kind: PlanStepKind::Migrate, + }); - plan.push(PlanStep::Up); + plan.push(PlanStep { + name_id: "up", + kind: PlanStepKind::Up, + }); + + // Start at the specified step + if let Some(start_at) = start_at { + let idx = plan + .iter() + .position(|x| x.name_id == start_at) + .ok_or_else(|| anyhow!("invalid start_at value: {}", start_at))?; + + plan = plan[idx..].to_vec(); + } Ok(plan) } /// List all of the Terraform plans in use for the generated plan. pub fn all_terraform_plans(ctx: &ProjectContext) -> Result> { - let plan_ids = build_plan(ctx)? + let plan_ids = build_plan(ctx, None)? .into_iter() .flat_map(|x| { - if let PlanStep::Terraform { plan_id, .. } = x { + if let PlanStepKind::Terraform { plan_id, .. } = x.kind { Some(plan_id) } else { None @@ -260,8 +327,11 @@ pub async fn execute_plan( for (i, step) in plan.iter().enumerate() { eprintln!(); eprintln!(); - rivet_term::status::info("Executing", format!("({}/{}) {step:?}", i + 1, plan.len())); - step.execute(ctx.clone(), &opts).await?; + rivet_term::status::info( + "Executing", + format!("({}/{}) {}", i + 1, plan.len(), step.name_id), + ); + step.kind.execute(ctx.clone(), &opts).await?; } Ok(()) @@ -277,8 +347,11 @@ pub async fn destroy_plan( for (i, step) in plan.iter().enumerate().rev() { eprintln!(); eprintln!(); - rivet_term::status::info("Destroying", format!("({}/{}) {step:?}", i + 1, plan.len())); - step.destroy(ctx.clone(), &opts).await?; + rivet_term::status::info( + "Destroying", + format!("({}/{}) {}", i + 1, plan.len(), step.name_id), + ); + step.kind.destroy(ctx.clone(), &opts).await?; } Ok(()) diff --git a/lib/bolt/core/src/tasks/test.rs b/lib/bolt/core/src/tasks/test.rs index efa24c78df..9b454314d2 100644 --- a/lib/bolt/core/src/tasks/test.rs +++ b/lib/bolt/core/src/tasks/test.rs @@ -277,6 +277,7 @@ async fn run_test(svc_ctx: &ServiceContext, test_name: Option<&str>) -> TestResu let mut cmd = Command::new("cargo"); cmd.current_dir(svc_ctx.path()); cmd.env("RUSTFLAGS", "--cfg tokio_unstable"); + cmd.env("CARGO_TARGET_DIR", project_ctx.path().join("target")); cmd.arg("test"); if let Some(jobs) = project_ctx.config_local().rust.num_jobs { cmd.arg("--jobs").arg(jobs.to_string()); diff --git a/lib/bolt/core/src/tasks/up.rs b/lib/bolt/core/src/tasks/up.rs index 833a65354a..fa37e0d46c 100644 --- a/lib/bolt/core/src/tasks/up.rs +++ b/lib/bolt/core/src/tasks/up.rs @@ -65,7 +65,7 @@ pub async fn up_services>( opts: UpOpts, ) -> Result> { let event = utils::telemetry::build_event(ctx, "bolt_up").await?; - utils::telemetry::capture_event(ctx, event).await?; + utils::telemetry::capture_event(ctx, event)?; // let run_context = RunContext::Service; let build_context = BuildContext::Bin { @@ -260,16 +260,21 @@ pub async fn up_services>( svc_ctx: svc_ctx.clone().clone(), build_context, driver: match &build_plan { - ServiceBuildPlan::ExistingLocalBuild { output_path } - | ServiceBuildPlan::BuildLocally { output_path } => { - derive_local_build_driver(svc_ctx, output_path.clone()).await + ServiceBuildPlan::ExistingLocalBuild { exec_path } + | ServiceBuildPlan::BuildLocally { exec_path } => { + derive_local_build_driver(svc_ctx, exec_path.clone()).await } ServiceBuildPlan::ExistingUploadedBuild { build_key: artifact_key, + exec_path, } | ServiceBuildPlan::BuildAndUpload { build_key: artifact_key, - } => derive_uploaded_svc_driver(svc_ctx, artifact_key.clone()).await, + exec_path, + } => { + derive_uploaded_svc_driver(svc_ctx, artifact_key.clone(), exec_path.clone()) + .await + } ServiceBuildPlan::Docker { image_tag } => ExecServiceDriver::Docker { image: image_tag.clone(), force_pull: false, @@ -395,12 +400,12 @@ async fn build_svc(svc_ctx: &ServiceContext, optimization: BuildOptimization) { async fn derive_local_build_driver( svc_ctx: &ServiceContext, - output_path: PathBuf, + exec_path: PathBuf, ) -> ExecServiceDriver { match &svc_ctx.config().runtime { RuntimeKind::Rust {} => ExecServiceDriver::LocalBinaryArtifact { // Convert path to be relative to the project root - path: output_path + exec_path: exec_path .strip_prefix(svc_ctx.project().await.path()) .expect("rust binary path not inside of project dir") .to_owned(), @@ -419,10 +424,12 @@ async fn derive_local_build_driver( async fn derive_uploaded_svc_driver( svc_ctx: &ServiceContext, artifact_key: String, + exec_path: String, ) -> ExecServiceDriver { match &svc_ctx.config().runtime { RuntimeKind::Rust {} => ExecServiceDriver::UploadedBinaryArtifact { artifact_key, + exec_path, args: Vec::new(), }, RuntimeKind::CRDB { .. } diff --git a/lib/bolt/core/src/utils/telemetry.rs b/lib/bolt/core/src/utils/telemetry.rs index 899e53d487..f2f868a2fb 100644 --- a/lib/bolt/core/src/utils/telemetry.rs +++ b/lib/bolt/core/src/utils/telemetry.rs @@ -92,9 +92,14 @@ pub async fn build_event(ctx: &ProjectContext, name: &str) -> Result Result<()> { +pub fn capture_event(ctx: &ProjectContext, event: async_posthog::Event) -> Result<()> { if !ctx.ns().rivet.telemetry.disable { - build_client().capture(event).await?; + tokio::spawn(async move { + match build_client().capture(event).await { + Ok(_) => {} + Err(err) => println!("Failed to capture event: {err:?}"), + } + }); } Ok(()) diff --git a/lib/cache/build/Cargo.toml b/lib/cache/build/Cargo.toml index 36ab1b569e..a84d4a99b0 100644 --- a/lib/cache/build/Cargo.toml +++ b/lib/cache/build/Cargo.toml @@ -15,7 +15,7 @@ rivet-pools = { path = "../../pools" } rivet-util = { path = "../../util/core" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" types = { path = "../../types/core" } uuid = { version = "1", features = ["v4"] } diff --git a/lib/chirp/client/Cargo.toml b/lib/chirp/client/Cargo.toml index f58f4dac4b..692b5e6f98 100644 --- a/lib/chirp/client/Cargo.toml +++ b/lib/chirp/client/Cargo.toml @@ -19,7 +19,7 @@ rivet-pools = { path = "../../pools" } rivet-util = { path = "../../util/core" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tokio-util = "0.6" tracing = "0.1" types = { path = "../../types/core" } diff --git a/lib/chirp/client/src/client.rs b/lib/chirp/client/src/client.rs index f6cd7ac023..09a79e8a8b 100644 --- a/lib/chirp/client/src/client.rs +++ b/lib/chirp/client/src/client.rs @@ -65,12 +65,12 @@ impl SharedClient { redis_cache: RedisPool, region: String, ) -> SharedClientHandle { - let spawn_res = tokio::task::Builder::new() - .name("chirp_client::metrics_update_update") - .spawn(metrics::start_update_uptime()); - if let Err(err) = spawn_res { - tracing::error!(?err, "failed to spawn user_presence_touch task"); - } + // let spawn_res = tokio::task::Builder::new() + // .name("chirp_client::metrics_update_update") + // .spawn(metrics::start_update_uptime()); + // if let Err(err) = spawn_res { + // tracing::error!(?err, "failed to spawn user_presence_touch task"); + // } Arc::new(SharedClient { nats, @@ -553,6 +553,12 @@ impl Client { .map_err(|x| ClientError::CreateSubscription(x.into()))?; // Manually publish message + tracing::trace!( + ?subject, + ?reply, + req_len = req_buf.len(), + "publishing rpc call to nats" + ); self.nats .publish_with_reply_and_headers( subject.clone(), @@ -768,86 +774,86 @@ impl Client { // Write message to Redis stream let span = self.perf().start(M::PERF_LABEL_WRITE_STREAM).await; - let mut backoff = rivet_util::Backoff::default_infinite(); - loop { - // Ignore for infinite backoff - backoff.tick().await; + // let mut backoff = rivet_util::Backoff::default_infinite(); + // loop { + // // Ignore for infinite backoff + // backoff.tick().await; - let mut conn = self.redis_chirp.clone(); - - let mut pipe = redis::pipe(); - pipe.atomic(); + let mut conn = self.redis_chirp.clone(); - // Write to stream - let topic_key = redis_keys::message_topic(M::NAME); - pipe.xadd_maxlen( - &topic_key, - redis::streams::StreamMaxlen::Approx(8192), - "*", - &[("m", &message_buf)], - ) - .ignore(); + let mut pipe = redis::pipe(); + pipe.atomic(); - // Write tails for all permuted parameters - for wildcard_parameters in &permuted_wildcard_parameters { - // Write tail - if let Some(ttl) = M::TAIL_TTL { - // Write single tail message + // Write to stream + let topic_key = redis_keys::message_topic(M::NAME); + pipe.xadd_maxlen( + &topic_key, + redis::streams::StreamMaxlen::Approx(8192), + "*", + &[("m", &message_buf)], + ) + .ignore(); + + // Write tails for all permuted parameters + for wildcard_parameters in &permuted_wildcard_parameters { + // Write tail + if let Some(ttl) = M::TAIL_TTL { + // Write single tail message + + let tail_key = redis_keys::message_tail::(wildcard_parameters); + + // Save message + pipe.hset( + &tail_key, + redis_keys::message_tail::REQUEST_ID, + req_id.to_string(), + ) + .ignore(); + pipe.hset(&tail_key, redis_keys::message_tail::TS, ts) + .ignore(); + pipe.hset(&tail_key, redis_keys::message_tail::BODY, message_buf) + .ignore(); - let tail_key = redis_keys::message_tail::(wildcard_parameters); + // Automatically expire + pipe.expire(&tail_key, ttl as usize).ignore(); + + // Write history + if M::HISTORY { + let history_key = redis_keys::message_history::(wildcard_parameters); + + // Remove old entries 10% of the time. + // + // This is a slow operation, so we perform this + // sparingly. + if rand::thread_rng().gen_bool(0.1) { + pipe.cmd("ZREMRANGEBYSCORE") + .arg(&history_key) + .arg("-inf") + .arg(rivet_util::timestamp::now() - ttl * 1000) + .ignore(); + } // Save message - pipe.hset( - &tail_key, - redis_keys::message_tail::REQUEST_ID, - req_id.to_string(), - ) - .ignore(); - pipe.hset(&tail_key, redis_keys::message_tail::TS, ts) - .ignore(); - pipe.hset(&tail_key, redis_keys::message_tail::BODY, message_buf) - .ignore(); + pipe.zadd(&history_key, message_buf, ts).ignore(); // Automatically expire - pipe.expire(&tail_key, ttl as usize).ignore(); - - // Write history - if M::HISTORY { - let history_key = redis_keys::message_history::(wildcard_parameters); - - // Remove old entries 10% of the time. - // - // This is a slow operation, so we perform this - // sparingly. - if rand::thread_rng().gen_bool(0.1) { - pipe.cmd("ZREMRANGEBYSCORE") - .arg(&history_key) - .arg("-inf") - .arg(rivet_util::timestamp::now() - ttl * 1000) - .ignore(); - } - - // Save message - pipe.zadd(&history_key, message_buf, ts).ignore(); - - // Automatically expire - pipe.expire(&history_key, ttl as usize).ignore(); - } else { - } + pipe.expire(&history_key, ttl as usize).ignore(); + } else { } } + } - // Write to Redis - match pipe.query_async::<_, ()>(&mut conn).await { - Ok(_) => { - tracing::debug!("write to redis stream succeeded"); - break; - } - Err(err) => { - tracing::error!(?err, "failed to write to redis"); - } + // Write to Redis + match pipe.query_async::<_, ()>(&mut conn).await { + Ok(_) => { + tracing::debug!("write to redis stream succeeded"); + // break; + } + Err(err) => { + tracing::error!(?err, "failed to write to redis"); } } + // } span.end(); } @@ -863,34 +869,39 @@ impl Client { // // Infinite backoff since we want to wait until the service reboots. let span = self.perf().start(M::PERF_LABEL_PUBLISH).await; - let mut backoff = rivet_util::Backoff::default_infinite(); - loop { - // Ignore for infinite backoff - backoff.tick().await; - - let nats_subject = nats_subject.to_owned(); - let message_buf = message_buf.to_vec(); - - if let Err(err) = self - .nats - .publish(nats_subject.clone(), message_buf.into()) - .await - { - tracing::warn!(?err, "publish message failed, trying again"); - continue; - } - - // TODO: Most messages don't need to be flushed immediately. We - // should add an option to enable high performance message - // publishing to enable flushing immediately after publishing. - // if let Err(err) = self.nats.flush().await { - // tracing::error!(?err, "flush message failed, the message probably sent"); - // break; - // } - - tracing::debug!("publish nats message succeeded"); - break; + // let mut backoff = rivet_util::Backoff::default_infinite(); + // loop { + // // Ignore for infinite backoff + // backoff.tick().await; + + let nats_subject = nats_subject.to_owned(); + let message_buf = message_buf.to_vec(); + + tracing::trace!( + ?nats_subject, + message_len = message_buf.len(), + "publishing message to nats" + ); + if let Err(err) = self + .nats + .publish(nats_subject.clone(), message_buf.into()) + .await + { + tracing::warn!(?err, "publish message failed, trying again"); + // continue; } + + // // TODO: Most messages don't need to be flushed immediately. We + // // should add an option to enable high performance message + // // publishing to enable flushing immediately after publishing. + // // if let Err(err) = self.nats.flush().await { + // // tracing::error!(?err, "flush message failed, the message probably sent"); + // // break; + // // } + + // tracing::debug!("publish nats message succeeded"); + // break; + // } span.end(); } diff --git a/lib/chirp/client/src/message.rs b/lib/chirp/client/src/message.rs index 92381f6848..bbe2df5205 100644 --- a/lib/chirp/client/src/message.rs +++ b/lib/chirp/client/src/message.rs @@ -22,6 +22,10 @@ pub fn serialize_message_params(parameters: &[impl AsRef], join: &str) -> S .map(|x| { if x == "*" { x.to_string() + } else if x.is_empty() { + // Provide a null value since NATS requires a valid subject + // TODO: Find a better null placeholder that doesn't conflict with manual value + "__NULL__".to_string() } else { urlencoding::encode(x).to_string() } @@ -35,11 +39,15 @@ where M: Message, S: AsRef, { - format!( - "chirp.msg.{}.{}", - M::NAME, - serialize_message_params(parameters, ".") - ) + if parameters.is_empty() { + format!("chirp.msg.{}", M::NAME) + } else { + format!( + "chirp.msg.{}.{}", + M::NAME, + serialize_message_params(parameters, ".") + ) + } } /// A message received from a Chirp subscription. diff --git a/lib/chirp/perf/Cargo.toml b/lib/chirp/perf/Cargo.toml index b4482c78b0..45a2e10feb 100644 --- a/lib/chirp/perf/Cargo.toml +++ b/lib/chirp/perf/Cargo.toml @@ -12,7 +12,7 @@ types = { path = "../../types/core" } rivet-metrics = { path = "../../metrics" } rivet-pools = { path = "../../pools" } thiserror = "1.0" -tokio = { version = "1.21", features = ["full", "tracing"] } +tokio = { version = "1.29", features = ["full", "tracing"] } tracing = "0.1" [dependencies.uuid] diff --git a/lib/chirp/worker/Cargo.toml b/lib/chirp/worker/Cargo.toml index 06e1211535..9f82ce0771 100644 --- a/lib/chirp/worker/Cargo.toml +++ b/lib/chirp/worker/Cargo.toml @@ -32,7 +32,7 @@ rivet-runtime = { path = "../../runtime" } rivet-util = { path = "../../util/core" } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21", features = ["full", "tracing"] } +tokio = { version = "1.29", features = ["full", "tracing"] } tracing = "0.1" types = { path = "../../types/core" } tracing-subscriber = { version = "0.3", optional = true, default-features = false, features = ["fmt", "json", "ansi"] } # Used with attribute diff --git a/lib/chirp/worker/src/manager.rs b/lib/chirp/worker/src/manager.rs index 1c7227ba69..340a507936 100644 --- a/lib/chirp/worker/src/manager.rs +++ b/lib/chirp/worker/src/manager.rs @@ -1,6 +1,6 @@ use chirp_metrics as metrics; use futures_util::StreamExt; -use global_error::{GlobalResult, GlobalError}; +use global_error::{GlobalError, GlobalResult}; use prost::Message; use redis::{self, AsyncCommands}; use rivet_connection::Connection; @@ -276,7 +276,7 @@ where .arg(pending_retry_time.as_millis() as i64) .arg("-") .arg("+") - .arg(16usize) + .arg(1usize) .query_async::<_, redis::streams::StreamPendingCountReply>(redis_chirp_conn) .await { @@ -451,7 +451,7 @@ where let read_options = redis::streams::StreamReadOptions::default() .group(&group, &consumer) .block(30_000) - .count(16); + .count(1); let res = match redis_chirp_conn .xread_options::<_, _, redis::streams::StreamReadReply>(keys, &[">"], &read_options) .await @@ -857,36 +857,42 @@ where self: Arc, req: &Request, ) -> GlobalResult { - // Will retry 4 times. This will take a maximum of 15 seconds. - let mut backoff = rivet_util::Backoff::new(5, Some(4), 1_000, 1_000); - loop { - let res = self - .worker - .handle(req.op_ctx()) - .instrument(tracing::info_span!("handle", name = %W::NAME)) - .await; - - // Attempt to retry the request with backoff - if matches!( - res, - Err(GlobalError::Internal { - retry_immediately: true, - .. - }) - ) { - tracing::info!("ticking request retry backoff"); - - if backoff.tick().await { - tracing::warn!("retry request failed too many times"); - return res; - } else { - tracing::info!("retrying request"); - } - } else { - // Return result immediately - return res; - } - } + self.worker + .handle(req.op_ctx()) + .instrument(tracing::info_span!("handle", name = %W::NAME)) + .await + + // // TODO: Add back + // // Will retry 4 times. This will take a maximum of 15 seconds. + // let mut backoff = rivet_util::Backoff::new(5, Some(4), 1_000, 1_000); + // loop { + // let res = self + // .worker + // .handle(req.op_ctx()) + // .instrument(tracing::info_span!("handle", name = %W::NAME)) + // .await; + + // // Attempt to retry the request with backoff + // if matches!( + // res, + // Err(GlobalError::Internal { + // retry_immediately: true, + // .. + // }) + // ) { + // tracing::info!("ticking request retry backoff"); + + // if backoff.tick().await { + // tracing::warn!("retry request failed too many times"); + // return res; + // } else { + // tracing::info!("retrying request"); + // } + // } else { + // // Return result immediately + // return res; + // } + // } } #[tracing::instrument(level = "trace", skip_all)] @@ -1028,11 +1034,11 @@ where #[tracing::instrument] async fn consumer_ack(self: Arc, msg_meta: RedisMessageMeta) { let mut backoff = rivet_util::Backoff::default(); - loop { - if backoff.tick().await { - tracing::error!("acking stream message failed too many times, aborting"); - return; - } + // loop { + // if backoff.tick().await { + // tracing::error!("acking stream message failed too many times, aborting"); + // return; + // } // Acknowledge the messages let mut redis_chirp = self.redis_chirp.clone(); @@ -1042,13 +1048,13 @@ where { Ok(_) => { tracing::info!(?msg_meta, "acknowledged stream message"); - break; + // break; } Err(err) => { tracing::error!(?err, "failed to ack message"); } } - } + // } } } diff --git a/lib/chirp/worker/src/test.rs b/lib/chirp/worker/src/test.rs index a6fe9b9502..18cca88a9e 100644 --- a/lib/chirp/worker/src/test.rs +++ b/lib/chirp/worker/src/test.rs @@ -74,8 +74,4 @@ impl TestCtx { pub async fn redis_user_presence(&self) -> Result { self.op_ctx.redis_user_presence().await } - - pub async fn redis_search(&self) -> Result { - self.op_ctx.redis_search().await - } } diff --git a/lib/connection/src/lib.rs b/lib/connection/src/lib.rs index 24633a1598..588f50390f 100644 --- a/lib/connection/src/lib.rs +++ b/lib/connection/src/lib.rs @@ -86,10 +86,6 @@ impl Connection { self.pools.redis("redis-user-presence") } - pub async fn redis_search(&self) -> Result { - self.pools.redis("redis-search") - } - pub fn perf(&self) -> &chirp_perf::PerfCtx { self.client.perf() } diff --git a/lib/convert/src/convert/game.rs b/lib/convert/src/convert/game.rs index 0ae85fa3af..4cd0c3cd54 100644 --- a/lib/convert/src/convert/game.rs +++ b/lib/convert/src/convert/game.rs @@ -10,13 +10,9 @@ pub fn handle(game: &backend::game::Game) -> GlobalResult { name_id: game.name_id.to_owned(), display_name: game.display_name.to_owned(), logo_url: util::route::game_logo( - game.logo_upload_id.map(|x| x.as_uuid()), - game.logo_file_name.as_ref(), - ), + &game), banner_url: util::route::game_banner( - game.banner_upload_id.map(|x| x.as_uuid()), - game.banner_file_name.as_ref(), - ), + &game), }) } @@ -36,13 +32,9 @@ pub fn summary( name_id: game.name_id.to_owned(), display_name: game.display_name.to_owned(), logo_url: util::route::game_logo( - game.logo_upload_id.map(|x| x.as_uuid()), - game.logo_file_name.as_ref(), - ), + &game), banner_url: util::route::game_banner( - game.banner_upload_id.map(|x| x.as_uuid()), - game.banner_file_name.as_ref(), - ), + &game), url: game_url, developer: Box::new(convert::group::handle(dev_team, true)?), }) diff --git a/lib/convert/src/convert/group.rs b/lib/convert/src/convert/group.rs index 00e733d153..95243c0339 100644 --- a/lib/convert/src/convert/group.rs +++ b/lib/convert/src/convert/group.rs @@ -12,10 +12,7 @@ pub fn handle(team: &backend::team::Team, is_developer: bool) -> GlobalResult, user_ids: Vec) -> GlobalResult { - let user_teams_res = op!([ctx] user_team_list { - user_ids: user_ids, - }) - .await?; - - let team_ids = user_teams_res - .users - .iter() - .map(|user| { - user.teams - .iter() - .map(|t| Ok(internal_unwrap_owned!(t.team_id))) - .collect::>>() - }) - .collect::>>()? - .into_iter() - .flatten() - .collect::>(); - - let (teams_res, dev_teams_res) = tokio::try_join!( - op!([ctx] team_get { - team_ids: team_ids.clone(), - }), - op!([ctx] team_dev_get { - team_ids: team_ids.clone(), - }), - )?; - - // TODO: hide all closed teams - let teams = teams_res.teams.clone(); - - Ok(TeamsCtx { - user_teams: user_teams_res, - teams, - dev_teams: dev_teams_res, - }) + return Ok(TeamsCtx { + user_teams: user::team_list::Response { users: vec![] }, + teams: vec![], + dev_teams: team_dev::get::Response { teams: vec![] }, + }); + + // let user_teams_res = op!([ctx] user_team_list { + // user_ids: user_ids, + // }) + // .await?; + + // let team_ids = user_teams_res + // .users + // .iter() + // .map(|user| { + // user.teams + // .iter() + // .map(|t| Ok(internal_unwrap_owned!(t.team_id))) + // .collect::>>() + // }) + // .collect::>>()? + // .into_iter() + // .flatten() + // .collect::>(); + + // let (teams_res, dev_teams_res) = tokio::try_join!( + // op!([ctx] team_get { + // team_ids: team_ids.clone(), + // }), + // op!([ctx] team_dev_get { + // team_ids: team_ids.clone(), + // }), + // )?; + + // // TODO: hide all closed teams + // let teams = teams_res.teams.clone(); + + // Ok(TeamsCtx { + // user_teams: user_teams_res, + // teams, + // dev_teams: dev_teams_res, + // }) } async fn follows( @@ -334,23 +340,24 @@ pub async fn mutual_follows( current_user_id: Uuid, raw_user_ids: Vec, ) -> GlobalResult { - // Converts to hashmap to remove duplicate queries - let queries = raw_user_ids - .clone() - .into_iter() - .flat_map(|user_id| [(current_user_id, user_id), (user_id, current_user_id)]) - .collect::>() - .into_iter() - .map(|(user_a_id, user_b_id)| user_follow::get::request::Query { - follower_user_id: Some(user_a_id.into()), - following_user_id: Some(user_b_id.into()), - }) - .collect::>(); - - op!([ctx] user_follow_get { - queries: queries, - }) - .await + return Ok(user_follow::get::Response { follows: vec![] }); + // // Converts to hashmap to remove duplicate queries + // let queries = raw_user_ids + // .clone() + // .into_iter() + // .flat_map(|user_id| [(current_user_id, user_id), (user_id, current_user_id)]) + // .collect::>() + // .into_iter() + // .map(|(user_a_id, user_b_id)| user_follow::get::request::Query { + // follower_user_id: Some(user_a_id.into()), + // following_user_id: Some(user_b_id.into()), + // }) + // .collect::>(); + + // op!([ctx] user_follow_get { + // queries: queries, + // }) + // .await } async fn linked_accounts( diff --git a/lib/convert/src/impls/cloud/mod.rs b/lib/convert/src/impls/cloud/mod.rs index 741ae4df14..715770edb0 100644 --- a/lib/convert/src/impls/cloud/mod.rs +++ b/lib/convert/src/impls/cloud/mod.rs @@ -2,9 +2,9 @@ use proto::{ backend::{self, pkg::*}, perf, }; +use rivet_api::models as new_models; use rivet_cloud_server::models; use rivet_operation::prelude::*; -use rivet_api::models as new_models; use crate::{ApiFrom, ApiTryFrom, ApiTryInto}; @@ -65,13 +65,9 @@ impl ApiTryFrom for models::GameHandle { name_id: value.name_id.to_owned(), display_name: value.display_name.to_owned(), logo_url: util::route::game_logo( - value.logo_upload_id.map(|x| *x), - value.logo_file_name.as_ref(), - ), + &value), banner_url: util::route::game_banner( - value.banner_upload_id.map(|x| *x), - value.banner_file_name.as_ref(), - ), + &value), }) } } diff --git a/lib/convert/src/impls/cloud/version/mod.rs b/lib/convert/src/impls/cloud/version/mod.rs index b351ccb732..a824be04e3 100644 --- a/lib/convert/src/impls/cloud/version/mod.rs +++ b/lib/convert/src/impls/cloud/version/mod.rs @@ -33,7 +33,8 @@ pub async fn config_to_proto( }, kv: value.kv.map(|_| backend::kv::VersionConfig {}), identity: value.identity.map(|x| (*x).try_into()).transpose()?, - module: todo!(), + // TODO: + module: None, }) } diff --git a/lib/health-checks/Cargo.toml b/lib/health-checks/Cargo.toml index 44a400cea8..886a1e5303 100644 --- a/lib/health-checks/Cargo.toml +++ b/lib/health-checks/Cargo.toml @@ -12,7 +12,7 @@ rivet-pools = { path = "../pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21", features = ["tracing"] } +tokio = { version = "1.29", features = ["tracing"] } tracing = "0.1" uuid = { version = "1", features = ["v4"] } diff --git a/lib/nomad-util/Cargo.toml b/lib/nomad-util/Cargo.toml index 4cc94c4770..4da6abae79 100644 --- a/lib/nomad-util/Cargo.toml +++ b/lib/nomad-util/Cargo.toml @@ -15,6 +15,6 @@ rivet-pools = { path = "../pools" } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["raw_value"] } thiserror = "1.0" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/lib/operation/core/Cargo.toml b/lib/operation/core/Cargo.toml index fb40076ee6..0cc3195145 100644 --- a/lib/operation/core/Cargo.toml +++ b/lib/operation/core/Cargo.toml @@ -23,6 +23,6 @@ rivet-pools = { path = "../../pools" } rivet-util = { path = "../../util/core" } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21", features = ["full", "tracing"] } +tokio = { version = "1.29", features = ["full", "tracing"] } tracing = "0.1" types = { path = "../../types/core" } diff --git a/lib/operation/core/src/lib.rs b/lib/operation/core/src/lib.rs index 05dad6afd8..cd28ffb480 100644 --- a/lib/operation/core/src/lib.rs +++ b/lib/operation/core/src/lib.rs @@ -90,7 +90,12 @@ where // TODO: Throw dedicated "timed out" error here // Process the request - let res = tokio::time::timeout(O::TIMEOUT, O::handle(self.wrap::(body)?)).await?; + let req_op_ctx = self.wrap::(body)?; + let timeout_fut = tokio::time::timeout(O::TIMEOUT, O::handle(req_op_ctx)); + let res = tokio::task::Builder::new() + .name("operation::handle") + .spawn(timeout_fut)? + .await??; // Record metrics { @@ -123,17 +128,18 @@ where .observe(dt); } - // Submit perf - let chirp = self.conn.chirp().clone(); - tokio::task::Builder::new().name("operation::perf").spawn( - async move { - // HACK: Force submit performance metrics after delay in order to ensure - // all spans have ended appropriately - tokio::time::sleep(Duration::from_secs(5)).await; - chirp.perf().submit().await; - } - .instrument(tracing::info_span!("operation_perf")), - )?; + // TODO: Add back + // // Submit perf + // let chirp = self.conn.chirp().clone(); + // tokio::task::Builder::new().name("operation::perf").spawn( + // async move { + // // HACK: Force submit performance metrics after delay in order to ensure + // // all spans have ended appropriately + // tokio::time::sleep(Duration::from_secs(5)).await; + // chirp.perf().submit().await; + // } + // .instrument(tracing::info_span!("operation_perf")), + // )?; res } @@ -285,10 +291,6 @@ where self.conn.redis_user_presence().await } - pub async fn redis_search(&self) -> Result { - self.conn.redis_search().await - } - pub fn perf(&self) -> &chirp_perf::PerfCtx { self.conn.perf() } diff --git a/lib/pools/Cargo.toml b/lib/pools/Cargo.toml index 74e89cc618..1800a2f206 100644 --- a/lib/pools/Cargo.toml +++ b/lib/pools/Cargo.toml @@ -13,20 +13,19 @@ lazy_static = "1.4" rand = "0.8" rivet-metrics = { path = "../metrics" } thiserror = "1.0" -tokio = { version = "1.21", features = ["tracing"] } +tokio = { version = "1.29", features = ["tracing"] } tokio-util = "0.6" tracing = "0.1" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false features = [ - "runtime-tokio-native-tls", + "runtime-tokio", "postgres", "macros", "uuid", "json", - "tls", ] [dependencies.redis] diff --git a/lib/pools/src/lib.rs b/lib/pools/src/lib.rs index c24645495b..1e280184df 100644 --- a/lib/pools/src/lib.rs +++ b/lib/pools/src/lib.rs @@ -75,7 +75,7 @@ async fn nats_from_env(client_name: String) -> Result, Error> { async move { match event { async_nats::Event::Connected => { - tracing::debug!(?server_addrs, "nats reconnected"); + tracing::info!(?server_addrs, "nats reconnected"); } async_nats::Event::Disconnected => { tracing::error!(?server_addrs, "nats disconnected"); @@ -100,12 +100,12 @@ async fn nats_from_env(client_name: String) -> Result, Error> { // NATS has built in backoff with jitter (with max of 4s), so // once the connection is established, we never have to worry // about disconnections that aren't handled by NATS. - tracing::debug!(?server_addrs, "nats connecting"); + tracing::info!(?server_addrs, "nats connecting"); let conn = options .connect(&server_addrs[..]) .await .map_err(Error::BuildNats)?; - tracing::debug!(?server_addrs, "nats connected"); + tracing::info!(?server_addrs, "nats connected"); Ok(Some(conn)) } else { @@ -120,40 +120,53 @@ fn crdb_from_env(client_name: String) -> Result, Error if let Some(svc_name_screaming) = key.strip_prefix("CRDB_URL_") { let svc_name = svc_name_screaming.to_lowercase().replace("_", "-"); - tracing::debug!(%url, "crdb creating connection"); + tracing::info!(%url, "crdb creating connection"); let client_name = client_name.clone(); let pool = sqlx::postgres::PgPoolOptions::new() // The default connection timeout is too high .acquire_timeout(Duration::from_secs(15)) + .max_lifetime(Duration::from_secs(60 * 5)) // Remove connections after a while in order to reduce load // on CRDB after bursts .idle_timeout(Some(Duration::from_secs(60))) // Open a connection // immediately on startup - .min_connections(1) + .min_connections(0) // Raise the cap, since this is effectively the amount of // simultaneous requests we can handle. See // https://www.cockroachlabs.com/docs/stable/connection-pooling.html - .max_connections(512) + .max_connections(20_000) // Speeds up requests at the expense of potential // failures // .test_before_acquire(false) - .after_connect({ - let url = url.clone(); - move |conn, _| { - let client_name = client_name.clone(); - let url = url.clone(); - Box::pin(async move { - tracing::debug!(%url, "crdb connected"); - sqlx::query("SET application_name = $1;") - .bind(&client_name) - .execute(conn) - .await?; - Ok(()) - }) - } - }) + // .after_connect(|conn, _meta| { + // Box::pin(async move { + // tracing::info!("pg connected"); + // Ok(()) + // }) + // }) + // .after_release(|conn, meta| { + // Box::pin(async move { + // tracing::info!("pg released"); + // Ok(false) + // }) + // }) + // .after_connect({ + // let url = url.clone(); + // move |conn, _| { + // let client_name = client_name.clone(); + // let url = url.clone(); + // Box::pin(async move { + // tracing::info!(%url, "crdb connected"); + // sqlx::query("SET application_name = $1;") + // .bind(&client_name) + // .execute(conn) + // .await?; + // Ok(()) + // }) + // } + // }) .connect_lazy(&url) .map_err(Error::BuildSqlx)?; @@ -177,13 +190,13 @@ async fn redis_from_env() -> Result, Error> { let pool = if let Some(existing) = existing_pools.get(&url) { existing.clone() } else { - tracing::debug!(%url, "redis connecting"); + tracing::info!(%url, "redis connecting"); let conn = redis::Client::open(url.as_str()) .map_err(Error::BuildRedis)? .get_tokio_connection_manager() .await .map_err(Error::BuildRedis)?; - tracing::debug!(%url, "redis connected"); + tracing::info!(%url, "redis connected"); conn }; diff --git a/lib/pools/src/pools.rs b/lib/pools/src/pools.rs index 619f30f265..a8d20f7ea8 100644 --- a/lib/pools/src/pools.rs +++ b/lib/pools/src/pools.rs @@ -100,6 +100,17 @@ impl PoolsInner { async fn record_metrics(&self) { use crate::metrics::*; + let mut dbs = self + .crdb_map() + .clone() + .into_iter() + .map(|(k, v)| (k, v.size(), v.num_idle())) + .filter(|x| x.1 > 0) + .collect::>(); + dbs.sort_by_key(|(_, size, _)| *size); + dbs.reverse(); + tracing::info!(?dbs, "db metrics"); + // CRDB for (db_name, pool) in self.crdb_map() { let label = &[db_name.as_str()]; diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml index 1b36d438e9..0989f3febd 100644 --- a/lib/runtime/Cargo.toml +++ b/lib/runtime/Cargo.toml @@ -10,7 +10,7 @@ console-subscriber = "0.1" lazy_static = "1.4" rivet-metrics = { path = "../metrics" } thiserror = "1.0" -tokio = { version = "1.21", features = ["full", "tracing"] } +tokio = { version = "1.29", features = ["full", "tracing"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } # Used with attribute diff --git a/lib/s3-util/Cargo.toml b/lib/s3-util/Cargo.toml index 79bd355205..1ae703da3a 100644 --- a/lib/s3-util/Cargo.toml +++ b/lib/s3-util/Cargo.toml @@ -13,5 +13,5 @@ aws-smithy-types = { version = "0.52" } http = "0.2" thiserror = "1.0" tracing = "0.1" -tokio = { version = "1.21" } +tokio = { version = "1.29" } diff --git a/lib/s3-util/src/lib.rs b/lib/s3-util/src/lib.rs index cd3bc66b5d..84f0142b0a 100644 --- a/lib/s3-util/src/lib.rs +++ b/lib/s3-util/src/lib.rs @@ -33,7 +33,7 @@ pub enum EndpointKind { External, } -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] pub enum Provider { Minio, Backblaze, diff --git a/lib/util/core/Cargo.toml b/lib/util/core/Cargo.toml index 6896abdaf1..4c1804f180 100644 --- a/lib/util/core/Cargo.toml +++ b/lib/util/core/Cargo.toml @@ -24,7 +24,7 @@ rivet-util-macros = { path = "../macros" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21", default-features = false, features = ["time", "sync", "fs", "parking_lot", "test-util", "macros"] } +tokio = { version = "1.29", default-features = false, features = ["time", "sync", "fs", "parking_lot", "test-util", "macros"] } types = { path = "../../types/core" } uuid = { version = "1", features = ["v4", "serde"] } diff --git a/lib/util/core/src/route.rs b/lib/util/core/src/route.rs index b526b17b44..13ff7c84f4 100644 --- a/lib/util/core/src/route.rs +++ b/lib/util/core/src/route.rs @@ -1,3 +1,5 @@ +use global_error::prelude::*; +use types::rivet::backend; use uuid::Uuid; use crate::env::{domain_main, origin_hub}; @@ -30,33 +32,44 @@ pub fn game_profile(game_name_id: &str) -> String { format!("{}/games/{}", origin_hub(), game_name_id) } -pub fn user_avatar(avatar_id: &str, upload_id: Option, file_name: Option<&String>) -> String { - if let (Some(upload_id), Some(file_name)) = (upload_id, file_name) { +pub fn user_avatar(user: &backend::user::User) -> String { + if let (Some(upload_id), Some(file_name), Some(provider)) = ( + user.profile_upload_id, + user.profile_file_name.as_ref(), + user.profile_provider, + ) { format!( - "https://media.{}/user-avatar/{}/{}", + "https://media.{}{}/user-avatar/{}/{}", domain_main(), + provider_str(provider), upload_id, file_name ) } else { - format!("https://assets.rivet.gg/avatars/{}.png", avatar_id) + format!("https://assets2.rivet.gg/avatars/{}.png", user.avatar_id) } } -pub fn custom_avatar(upload_id: Uuid, file_name: &str) -> String { +pub fn custom_avatar(upload_id: Uuid, file_name: &str, provider: i32) -> String { format!( - "https://media.{}/user-avatar/{}/{}", + "https://media.{}/{}/user-avatar/{}/{}", domain_main(), + provider_str(provider), upload_id, file_name ) } -pub fn team_avatar(upload_id: Option, file_name: Option<&String>) -> Option { - if let (Some(upload_id), Some(file_name)) = (upload_id, file_name) { +pub fn team_avatar(team: &backend::team::Team) -> Option { + if let (Some(upload_id), Some(file_name), Some(provider)) = ( + team.profile_upload_id, + team.profile_file_name.as_ref(), + team.profile_provider, + ) { Some(format!( - "https://media.{}/team-avatar/{}/{}", + "https://media.{}/{}/team-avatar/{}/{}", domain_main(), + provider_str(provider), upload_id, file_name )) @@ -65,11 +78,16 @@ pub fn team_avatar(upload_id: Option, file_name: Option<&String>) -> Optio } } -pub fn game_logo(upload_id: Option, file_name: Option<&String>) -> Option { - if let (Some(upload_id), Some(file_name)) = (upload_id, file_name) { +pub fn game_logo(game: &backend::game::Game) -> Option { + if let (Some(upload_id), Some(file_name), Some(provider)) = ( + game.logo_upload_id, + game.logo_file_name.as_ref(), + game.logo_provider, + ) { Some(format!( - "https://media.{}/game-logo/{}/{}", + "https://media.{}/{}/game-logo/{}/{}", domain_main(), + provider_str(provider), upload_id, file_name )) @@ -78,11 +96,16 @@ pub fn game_logo(upload_id: Option, file_name: Option<&String>) -> Option< } } -pub fn game_banner(upload_id: Option, file_name: Option<&String>) -> Option { - if let (Some(upload_id), Some(file_name)) = (upload_id, file_name) { +pub fn game_banner(game: &backend::game::Game) -> Option { + if let (Some(upload_id), Some(file_name), Some(provider)) = ( + game.banner_upload_id, + game.banner_file_name.as_ref(), + game.banner_provider, + ) { Some(format!( - "https://media.{}/game-banner/{}/{}", + "https://media.{}/{}/game-banner/{}/{}", domain_main(), + provider_str(provider), upload_id, file_name )) @@ -96,9 +119,18 @@ pub fn identity_game_link(link_token: &str) -> String { } pub fn cloud_device_link(link_token: &str) -> String { - format!("{}/developer/devices/link/{}", origin_hub(), link_token) + format!("{}/devices/link/{}", origin_hub(), link_token) } pub fn team_billing(team_id: Uuid) -> String { format!("{}/groups/{}/billing", origin_hub(), team_id) } + +fn provider_str(provider: i32) -> &'static str { + // Default gracefully + match backend::upload::Provider::from_i32(provider).unwrap_or_default() { + backend::upload::Provider::Minio => "minio", + backend::upload::Provider::Backblaze => "backblaze", + backend::upload::Provider::Aws => "aws", + } +} diff --git a/lib/util/env/src/lib.rs b/lib/util/env/src/lib.rs index 2b3d6dee62..6f04d441e8 100644 --- a/lib/util/env/src/lib.rs +++ b/lib/util/env/src/lib.rs @@ -5,17 +5,17 @@ pub async fn read_secret(key: &[impl AsRef]) -> Result]) -> Result, std::env::VarError> { +pub async fn read_secret_opt( + key: &[impl AsRef], +) -> Result, std::env::VarError> { let env_var = read_secret(key).await; - + match env_var { Ok(v) => Ok(Some(v)), - Err(var_error) => { - match var_error { - std::env::VarError::NotPresent => Ok(None), - std::env::VarError::NotUnicode(_) => Err(var_error), - } - } + Err(var_error) => match var_error { + std::env::VarError::NotPresent => Ok(None), + std::env::VarError::NotUnicode(_) => Err(var_error), + }, } } diff --git a/proto/backend/game.proto b/proto/backend/game.proto index 58f7a076d6..4c5849d588 100644 --- a/proto/backend/game.proto +++ b/proto/backend/game.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package rivet.backend.game; import "proto/common.proto"; +import "proto/backend/upload.proto"; message Game { rivet.common.Uuid game_id = 1; @@ -13,10 +14,13 @@ message Game { rivet.common.Uuid developer_team_id = 6; string description = 7; repeated string tags = 8; + optional rivet.common.Uuid logo_upload_id = 9; optional string logo_file_name = 10; + optional upload.Provider logo_provider = 15; optional rivet.common.Uuid banner_upload_id = 11; optional string banner_file_name = 12; + optional upload.Provider banner_provider = 16; optional string plan_code = 13; optional rivet.common.Uuid subscription_id = 14; diff --git a/proto/backend/team.proto b/proto/backend/team.proto index 00794eec2a..6342377299 100644 --- a/proto/backend/team.proto +++ b/proto/backend/team.proto @@ -4,6 +4,7 @@ package rivet.backend.team; import "proto/common.proto"; import "proto/backend/user.proto"; +import "proto/backend/upload.proto"; message Team { reserved 5, 8; @@ -14,6 +15,7 @@ message Team { string bio = 4; optional rivet.common.Uuid profile_upload_id = 9; optional string profile_file_name = 10; + optional upload.Provider profile_provider = 11; int64 create_ts = 6; Publicity publicity = 7; } diff --git a/proto/backend/user.proto b/proto/backend/user.proto index 6e3471c966..9004b243e2 100644 --- a/proto/backend/user.proto +++ b/proto/backend/user.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package rivet.backend.user; import "proto/common.proto"; +import "proto/backend/upload.proto"; message User { reserved 5, 9; @@ -13,6 +14,7 @@ message User { string avatar_id = 4; optional rivet.common.Uuid profile_upload_id = 10; optional string profile_file_name = 11; + optional upload.Provider profile_provider = 14; int64 join_ts = 6; string bio = 7; bool is_admin = 8; diff --git a/shell.nix b/shell.nix index fbcc93de4a..a6d47230d0 100644 --- a/shell.nix +++ b/shell.nix @@ -46,7 +46,7 @@ in redis # For the redis-cli # Runtimes - nodejs-slim # Required for Fern + nodejs # Required for Fern # Compilers clang diff --git a/svc/Cargo.lock b/svc/Cargo.lock index 77caa4870a..df77919924 100644 --- a/svc/Cargo.lock +++ b/svc/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -26,6 +35,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", ] @@ -41,9 +51,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "analytics-worker" @@ -80,9 +90,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "api-admin" @@ -907,7 +917,7 @@ dependencies = [ "bytes", "futures", "http", - "itoa 1.0.6", + "itoa 1.0.9", "memchr", "nkeys", "nuid", @@ -922,7 +932,7 @@ dependencies = [ "serde_nanos", "serde_repr", "thiserror", - "time 0.3.22", + "time 0.3.25", "tokio", "tokio-retry", "tracing", @@ -935,7 +945,7 @@ version = "0.24.0-ALPHA.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cdefe54cd7867d937c0a507d2a3a830af410044282cd3e4002b5b7860e1892e" dependencies = [ - "rustls 0.21.2", + "rustls 0.21.6", "tokio", "webpki 0.22.0", ] @@ -955,13 +965,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -972,9 +982,9 @@ checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82" [[package]] name = "atoi" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ "num-traits", ] @@ -1038,7 +1048,7 @@ dependencies = [ "aws-types", "bytes", "bytes-utils", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "tokio-stream", @@ -1077,7 +1087,7 @@ dependencies = [ "percent-encoding", "regex", "sha2 0.10.7", - "time 0.3.22", + "time 0.3.25", "tracing", ] @@ -1137,7 +1147,7 @@ dependencies = [ "aws-smithy-http-tower 0.41.0", "aws-smithy-types 0.41.0", "bytes", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "hyper", @@ -1161,7 +1171,7 @@ dependencies = [ "aws-smithy-http-tower 0.52.0", "aws-smithy-types 0.52.0", "bytes", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "hyper", @@ -1274,10 +1284,10 @@ version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c7f957a2250cc0fa4ccf155e00aeac9a81f600df7cd4ecc910c75030e6534f5" dependencies = [ - "itoa 1.0.6", + "itoa 1.0.9", "num-integer", "ryu", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -1287,10 +1297,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "987b1e37febb9bd409ca0846e82d35299e572ad8279bc404778caeb5fc05ad56" dependencies = [ "base64-simd", - "itoa 1.0.6", + "itoa 1.0.9", "num-integer", "ryu", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -1320,19 +1330,19 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", "http-body", "hyper", - "itoa 1.0.6", + "itoa 1.0.9", "matchit", "memchr", "mime", @@ -1363,6 +1373,21 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -1395,9 +1420,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.1.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bcrypt" @@ -1423,6 +1448,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +dependencies = [ + "serde", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -1458,15 +1492,15 @@ dependencies = [ "heck 0.3.3", "serde", "serde_json", - "toml 0.7.4", + "toml 0.7.6", "uuid", ] [[package]] name = "bstr" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", ] @@ -1622,9 +1656,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" +dependencies = [ + "libc", +] [[package]] name = "cdn-namespace-auth-user-remove" @@ -2259,7 +2296,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -2544,9 +2581,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ab2224a0311582eb03adba4caaf18644f7b1f10a760803a803b9b605187fc7" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" dependencies = [ "console-api", "crossbeam-channel", @@ -2582,6 +2619,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b" +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + [[package]] name = "core-foundation" version = "0.9.3" @@ -2600,18 +2643,33 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + [[package]] name = "crc32c" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" +checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" dependencies = [ "rustc_version", ] @@ -2671,7 +2729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", - "itoa 1.0.6", + "itoa 1.0.9", "ryu", "serde", ] @@ -2738,9 +2796,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ "darling_core", "darling_macro", @@ -2748,27 +2806,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "darling_macro" -version = "0.20.1" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -2797,47 +2855,48 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4" dependencies = [ - "const-oid", + "const-oid 0.6.2", ] [[package]] -name = "digest" -version = "0.9.0" +name = "der" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ - "generic-array", + "const-oid 0.9.5", + "pem-rfc7468 0.7.0", + "zeroize", ] [[package]] -name = "digest" -version = "0.10.7" +name = "deranged" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" dependencies = [ - "block-buffer 0.10.4", - "crypto-common", - "subtle", + "serde", ] [[package]] -name = "dirs" -version = "4.0.0" +name = "digest" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "dirs-sys", + "generic-array", ] [[package]] -name = "dirs-sys" -version = "0.3.7" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "libc", - "redox_users", - "winapi", + "block-buffer 0.10.4", + "const-oid 0.9.5", + "crypto-common", + "subtle", ] [[package]] @@ -2852,7 +2911,7 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" dependencies = [ - "signature", + "signature 1.6.4", ] [[package]] @@ -2869,9 +2928,12 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "email-address-parser" @@ -2945,24 +3007,30 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "erased-serde" -version = "0.3.25" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" +checksum = "da96524cc884f6558f1769b6c46686af2fe8e8b4cd253bd5a3cdba8181b8e070" dependencies = [ "serde", ] [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2975,6 +3043,17 @@ dependencies = [ "libc", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -3166,6 +3245,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "fcm" version = "0.9.2" @@ -3196,6 +3281,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3288,13 +3385,13 @@ dependencies = [ [[package]] name = "futures-intrusive" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.11.2", + "parking_lot", ] [[package]] @@ -3311,7 +3408,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -3792,6 +3889,12 @@ dependencies = [ "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "global-error" version = "0.1.5" @@ -3818,9 +3921,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -3828,7 +3931,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util 0.7.8", @@ -3883,7 +3986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", - "bitflags", + "bitflags 1.3.2", "bytes", "headers-core", "http", @@ -3921,18 +4024,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -3958,6 +4052,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -3966,7 +4069,7 @@ checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.6", + "itoa 1.0.9", ] [[package]] @@ -4000,9 +4103,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -4013,7 +4116,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.6", + "itoa 1.0.9", "pin-project-lite", "socket2", "tokio", @@ -4057,13 +4160,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" dependencies = [ + "futures-util", "http", "hyper", - "rustls 0.21.2", + "rustls 0.21.6", "tokio", "tokio-rustls 0.24.1", ] @@ -4203,6 +4307,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "indoc" version = "1.0.9" @@ -4227,17 +4341,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ip-info" version = "0.0.1" @@ -4255,9 +4358,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "itertools" @@ -4276,9 +4379,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "job-gc" @@ -4576,12 +4679,32 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libm" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "libsqlite3-sys" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "linked-hash-map" @@ -4591,9 +4714,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -4637,14 +4760,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "md-5" @@ -4700,7 +4823,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -5073,6 +5196,7 @@ dependencies = [ "game-namespace-get", "game-namespace-version-set", "game-version-get", + "heck 0.3.3", "http", "job-run-get", "lazy_static", @@ -5416,6 +5540,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -5426,25 +5567,46 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", + "libm", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -5459,11 +5621,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -5480,7 +5642,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -5491,9 +5653,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -5513,17 +5675,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -5531,21 +5682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -5556,16 +5693,16 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", "windows-targets", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pem" @@ -5578,9 +5715,18 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.2.4" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f22eb0e3c593294a99e9ff4b24cf6b752d43f193aa4415fe5077c159996d497" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e93a3b1cc0510b03020f33f21e62acdde3dcaef432edc95bea377fbd4c2cd4" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ "base64ct", ] @@ -5603,9 +5749,9 @@ dependencies = [ [[package]] name = "pest" -version = "2.6.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16833386b02953ca926d19f64af613b9bf742c48dcd5e09b32fbfc9740bf84e2" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ "thiserror", "ucd-trie", @@ -5613,9 +5759,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.6.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7763190f9406839f99e5197afee8c9e759969f7dbfa40ad3b8dbee8757b745b5" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -5623,22 +5769,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.6.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249061b22e99973da1f5f5f1410284419e283bb60b79255bf5f42a94b66a2e00" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "pest_meta" -version = "2.6.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457c310cfc9cf3f22bc58901cc7f0d3410ac5d6298e432a4f9a6138565cb6df6" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -5652,34 +5798,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c" [[package]] name = "pin-utils" @@ -5687,18 +5833,39 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + [[package]] name = "pkcs8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447" dependencies = [ - "der", - "pem-rfc7468", - "spki", + "der 0.4.5", + "pem-rfc7468 0.2.3", + "spki 0.4.1", "zeroize", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.8", + "spki 0.7.2", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -5799,9 +5966,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -5828,7 +5995,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot 0.12.1", + "parking_lot", "protobuf", "thiserror", ] @@ -5951,9 +6118,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -6006,7 +6173,7 @@ dependencies = [ "combine", "futures", "futures-util", - "itoa 1.0.6", + "itoa 1.0.9", "native-tls", "percent-encoding", "pin-project-lite", @@ -6027,44 +6194,25 @@ dependencies = [ "regex", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -6076,6 +6224,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.4", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6084,9 +6243,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "region-get" @@ -6158,7 +6317,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.0", + "hyper-rustls 0.24.1", "hyper-tls", "ipnet", "js-sys", @@ -6169,7 +6328,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.2", + "rustls 0.21.6", "rustls-pemfile", "serde", "serde_json", @@ -6197,7 +6356,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -6570,7 +6729,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -6781,6 +6940,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "rsa" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" +dependencies = [ + "byteorder", + "const-oid 0.9.5", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1", + "pkcs8 0.10.2", + "rand_core 0.6.4", + "signature 2.1.0", + "spki 0.7.2", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -6792,16 +6979,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.38.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" dependencies = [ - "bitflags", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -6831,9 +7017,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.2" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", @@ -6867,18 +7053,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64 0.21.2", ] [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.101.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" dependencies = [ "ring", "untrusted", @@ -6886,15 +7072,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "s3-util" @@ -6912,11 +7098,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -6931,9 +7117,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -6969,11 +7155,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -6982,9 +7168,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -6992,15 +7178,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "bdb30a74471f5b7a1fa299f40b4bf1be93af61116df95465b2b5fc419331e430" dependencies = [ "serde_derive", ] @@ -7016,13 +7202,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "6f4c2c6ea4bc09b5c419012eafcdb0fcef1d9119d626c8f3a0708a5b92d38a70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -7038,11 +7224,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "itoa 1.0.6", + "itoa 1.0.9", "ryu", "serde", ] @@ -7058,20 +7244,20 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "serde_spanned" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -7083,7 +7269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.6", + "itoa 1.0.9", "ryu", "serde", ] @@ -7097,11 +7283,11 @@ dependencies = [ "base64 0.13.1", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "serde_with_macros", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -7113,7 +7299,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -7190,9 +7376,9 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfecc059e81632eef1dd9b79e22fc28b8fe69b30d3357512a77a0ad8ee3c782" dependencies = [ - "pkcs8", + "pkcs8 0.7.6", "rand_core 0.6.4", - "signature", + "signature 1.6.4", "zeroize", ] @@ -7202,6 +7388,16 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +[[package]] +name = "signature" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd-abstraction" version = "0.7.1" @@ -7220,7 +7416,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.22", + "time 0.3.25", ] [[package]] @@ -7234,9 +7430,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" @@ -7254,13 +7450,32 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32" dependencies = [ - "der", + "der 0.4.5", +] + +[[package]] +name = "spki" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +dependencies = [ + "base64ct", + "der 0.7.8", ] [[package]] @@ -7276,94 +7491,203 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" dependencies = [ "sqlx-core", "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", ] [[package]] name = "sqlx-core" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ - "ahash 0.7.6", + "ahash 0.8.3", "atoi", - "base64 0.13.1", "bit-vec", - "bitflags", "byteorder", "bytes", + "crc", "crossbeam-queue", - "dirs", "dotenvy", "either", "event-listener", "futures-channel", "futures-core", "futures-intrusive", + "futures-io", "futures-util", "hashlink", "hex", - "hkdf", - "hmac", - "indexmap", - "itoa 1.0.6", - "libc", + "indexmap 2.0.0", "log", - "md-5", "memchr", "once_cell", "paste", "percent-encoding", - "rand", "serde", "serde_json", - "sha1 0.10.5", "sha2 0.10.7", "smallvec", "sqlformat", - "sqlx-rt", - "stringprep", "thiserror", + "tokio", "tokio-stream", + "tracing", "url", "uuid", - "whoami", ] [[package]] name = "sqlx-macros" -version = "0.6.3" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" dependencies = [ "dotenvy", "either", "heck 0.4.1", + "hex", "once_cell", "proc-macro2", "quote", + "serde", "serde_json", + "sha2 0.10.7", "sqlx-core", - "sqlx-rt", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", "syn 1.0.109", + "tempfile", + "tokio", "url", ] [[package]] -name = "sqlx-rt" -version = "0.6.3" +name = "sqlx-mysql" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" +checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ - "native-tls", + "atoi", + "base64 0.21.2", + "bitflags 2.3.3", + "byteorder", + "bytes", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa 1.0.9", + "log", + "md-5", + "memchr", "once_cell", - "tokio", - "tokio-native-tls", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1 0.10.5", + "sha2 0.10.7", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" +dependencies = [ + "atoi", + "base64 0.21.2", + "bit-vec", + "bitflags 2.3.3", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa 1.0.9", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1 0.10.5", + "sha2 0.10.7", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "uuid", ] [[package]] @@ -7374,9 +7698,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -7429,9 +7753,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -7811,36 +8135,35 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ - "autocfg", "cfg-if", - "fastrand", - "redox_syscall 0.3.5", + "fastrand 2.0.0", + "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -7876,11 +8199,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ - "itoa 1.0.6", + "deranged", + "itoa 1.0.9", "serde", "time-core", "time-macros", @@ -7894,9 +8218,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -7973,22 +8297,23 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -8009,7 +8334,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -8061,7 +8386,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.2", + "rustls 0.21.6", "tokio", ] @@ -8115,9 +8440,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", "serde_spanned", @@ -8127,20 +8452,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -8183,7 +8508,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -8222,13 +8547,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8803eee176538f94ae9a14b55b2804eb7e1441f8210b1c31290b3bccdccff73b" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -8318,14 +8643,14 @@ dependencies = [ "regex", "schemac", "serde", - "toml 0.7.4", + "toml 0.7.6", ] [[package]] name = "ucd-trie" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicase" @@ -8344,9 +8669,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -8507,9 +8832,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "user-avatar-upload-complete" @@ -8949,9 +9274,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.4" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom", "serde", @@ -9017,7 +9342,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -9051,7 +9376,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9127,13 +9452,9 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" -dependencies = [ - "wasm-bindgen", - "web-sys", -] +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" [[package]] name = "winapi" @@ -9166,21 +9487,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -9192,97 +9498,55 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -9291,9 +9555,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.7" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" +checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64" dependencies = [ "memchr", ] @@ -9339,5 +9603,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] diff --git a/svc/api/admin/Cargo.toml b/svc/api/admin/Cargo.toml index b113ac624d..885dae3e39 100644 --- a/svc/api/admin/Cargo.toml +++ b/svc/api/admin/Cargo.toml @@ -27,7 +27,7 @@ s3-util = { path = "../../../lib/s3-util" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-futures = "0.2" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } diff --git a/svc/api/admin/src/auth.rs b/svc/api/admin/src/auth.rs index d7313b00e5..44c06adb3c 100644 --- a/svc/api/admin/src/auth.rs +++ b/svc/api/admin/src/auth.rs @@ -23,9 +23,11 @@ impl ApiAuth for Auth { API_FORBIDDEN, reason = "Invalid auth" ); - } - Ok(Auth { _claims: None }) + Ok(Auth { _claims: None }) + } else { + internal_panic!("unreachable"); + } } async fn rate_limit(_rate_limit_ctx: AuthRateLimitCtx<'_>) -> GlobalResult<()> { diff --git a/svc/api/auth/Cargo.toml b/svc/api/auth/Cargo.toml index 8a6d905749..9b167b29cc 100644 --- a/svc/api/auth/Cargo.toml +++ b/svc/api/auth/Cargo.toml @@ -24,7 +24,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/cf-verification/Cargo.toml b/svc/api/cf-verification/Cargo.toml index ceb3b9e93a..6d267634f7 100644 --- a/svc/api/cf-verification/Cargo.toml +++ b/svc/api/cf-verification/Cargo.toml @@ -24,7 +24,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/chat/Cargo.toml b/svc/api/chat/Cargo.toml index 84b210c7c8..0b3fa8dfad 100644 --- a/svc/api/chat/Cargo.toml +++ b/svc/api/chat/Cargo.toml @@ -25,7 +25,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/chat/src/convert/identity.rs b/svc/api/chat/src/convert/identity.rs index 65bd0f94bf..1c50a22c10 100644 --- a/svc/api/chat/src/convert/identity.rs +++ b/svc/api/chat/src/convert/identity.rs @@ -13,11 +13,7 @@ pub fn handle_without_presence( identity_id: user_id.to_string(), display_name: user.display_name.to_owned(), account_number: user.account_number as i32, - avatar_url: util::route::user_avatar( - &user.avatar_id, - user.profile_upload_id.map(|x| x.as_uuid()), - user.profile_file_name.as_ref(), - ), + avatar_url: util::route::user_avatar(&user), presence: None, is_registered: true, // TODO: external: models::IdentityExternalLinks { diff --git a/svc/api/chat/src/convert/mod.rs b/svc/api/chat/src/convert/mod.rs index fccc853c95..c2c9fd80fa 100644 --- a/svc/api/chat/src/convert/mod.rs +++ b/svc/api/chat/src/convert/mod.rs @@ -18,14 +18,8 @@ pub fn game_handle(game: &backend::game::Game) -> GlobalResult>>()?, + )) + } else { + None + }; + + let image_presigned_requests = if multipart_upload { + Some( + create_res + .image_presigned_requests + .iter() + .cloned() + .map(ApiTryInto::try_into) + .collect::>>()?, + ) + } else { + None + }; + + Ok(new_models::CloudGamesCreateGameBuildResponse { + build_id: internal_unwrap!(create_res.build_id).as_uuid(), + upload_id: internal_unwrap!(create_res.upload_id).as_uuid(), + image_presigned_request, + image_presigned_requests, }) } diff --git a/svc/api/group/Cargo.toml b/svc/api/group/Cargo.toml index c5791e5fc2..9bbb657332 100644 --- a/svc/api/group/Cargo.toml +++ b/svc/api/group/Cargo.toml @@ -25,7 +25,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/group/src/convert/group.rs b/svc/api/group/src/convert/group.rs index 6005200ab1..7681bedbcb 100644 --- a/svc/api/group/src/convert/group.rs +++ b/svc/api/group/src/convert/group.rs @@ -9,10 +9,7 @@ pub fn handle(team: &backend::team::Team, is_developer: bool) -> GlobalResult GlobalResult>(), 1024)?; - let thumbnail_url = util::route::user_avatar( - &sender.avatar_id, - sender.profile_upload_id.map(|id| id.as_uuid()), - sender.profile_file_name.as_ref(), - ); + let thumbnail_url = util::route::user_avatar(&sender); let url = util::route::thread(thread_id.as_uuid()); Ok(Some(models::IdentityGlobalEventNotification { @@ -81,14 +77,16 @@ pub async fn get_namespace_id(ctx: &Ctx) -> GlobalResult { pub async fn resolve_user_with_game_user_id( ctx: &Ctx, game_user_id: Uuid, -) -> GlobalResult { +) -> GlobalResult> { let game_user_res = op!([ctx] game_user_get { game_user_ids: vec![game_user_id.into()] }) .await?; - let game_user = internal_unwrap_owned!(game_user_res.game_users.first()).clone(); + let Some(game_user) = game_user_res.game_users.first().clone() else { + return Ok(None) + }; - Ok(internal_unwrap!(game_user.user_id).as_uuid()) + Ok(Some(internal_unwrap!(game_user.user_id).as_uuid())) } pub fn touch_user_presence(ctx: OperationContext<()>, user_id: Uuid, silent: bool) { diff --git a/svc/api/job/Cargo.toml b/svc/api/job/Cargo.toml index 8e809daf93..b53305d27b 100644 --- a/svc/api/job/Cargo.toml +++ b/svc/api/job/Cargo.toml @@ -23,7 +23,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/kv/Cargo.toml b/svc/api/kv/Cargo.toml index c315b12f04..2af83b60a0 100644 --- a/svc/api/kv/Cargo.toml +++ b/svc/api/kv/Cargo.toml @@ -26,7 +26,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/kv/src/route/mod.rs b/svc/api/kv/src/route/mod.rs index a07de78553..d849fd9a94 100644 --- a/svc/api/kv/src/route/mod.rs +++ b/svc/api/kv/src/route/mod.rs @@ -22,17 +22,59 @@ define_router! { cors: CorsConfigBuilder::public().build(), routes: { "entries": { - GET: operations::get(query: operations::SingleQuery), - PUT: operations::put(body: models::KvPutRequest), - DELETE: operations::delete(query: operations::SingleQuery), + GET: operations::get( + query: operations::SingleQuery, + rate_limit: { + buckets: [ + { count: 1_000_000 }, + ], + }, + ), + PUT: operations::put( + body: models::KvPutRequest, + rate_limit: { + buckets: [ + { count: 100_000 }, + ], + }, + ), + DELETE: operations::delete( + query: operations::SingleQuery, + rate_limit: { + buckets: [ + { count: 100_000 }, + ], + }, + ), }, "entries" / "list": { GET: operations::list(query: operations::ListQuery), }, "entries" / "batch": { - GET: batch_operations::get_batch(query: batch_operations::BatchQuery), - PUT: batch_operations::put_batch(body: models::KvPutBatchRequest), - DELETE: batch_operations::delete_batch(query: batch_operations::BatchQuery), + GET: batch_operations::get_batch( + query: batch_operations::BatchQuery, + rate_limit: { + buckets: [ + { count: 1_000_000 }, + ], + }, + ), + PUT: batch_operations::put_batch( + body: models::KvPutBatchRequest, + rate_limit: { + buckets: [ + { count: 100_000 }, + ], + }, + ), + DELETE: batch_operations::delete_batch( + query: batch_operations::BatchQuery, + rate_limit: { + buckets: [ + { count: 100_000 }, + ], + }, + ), }, }, } diff --git a/svc/api/matchmaker/Cargo.toml b/svc/api/matchmaker/Cargo.toml index 10dfbd2ade..1d52cfa730 100644 --- a/svc/api/matchmaker/Cargo.toml +++ b/svc/api/matchmaker/Cargo.toml @@ -25,7 +25,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/matchmaker/tests/basic.rs b/svc/api/matchmaker/tests/basic.rs index 3937db13b5..112c7dc025 100644 --- a/svc/api/matchmaker/tests/basic.rs +++ b/svc/api/matchmaker/tests/basic.rs @@ -170,7 +170,13 @@ impl Ctx { regions: vec![backend::matchmaker::lobby_group::Region { region_id: Some(region_id.into()), tier_name_id: "basic-1d8".into(), - idle_lobbies: None, + idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies { + min_idle_lobbies: 0, + // Set a high max lobby count in case this is + // coming from a test that test mm-lobby-create + // without creating an associated player + max_idle_lobbies: 32, + }), }], max_players_normal: 8, max_players_direct: 10, diff --git a/svc/api/module/Cargo.toml b/svc/api/module/Cargo.toml index 613197bf4e..cb5006ff0e 100644 --- a/svc/api/module/Cargo.toml +++ b/svc/api/module/Cargo.toml @@ -26,7 +26,7 @@ rivet-pools = { path = "../../../lib/pools" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/portal/Cargo.toml b/svc/api/portal/Cargo.toml index 548f8b1888..000165a2e4 100644 --- a/svc/api/portal/Cargo.toml +++ b/svc/api/portal/Cargo.toml @@ -26,7 +26,7 @@ rivet-portal-server = { path = "../../../lib/smithy-output/api-portal/rust-serve serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/portal/src/build.rs b/svc/api/portal/src/build.rs index bca42cee26..627fcb5981 100644 --- a/svc/api/portal/src/build.rs +++ b/svc/api/portal/src/build.rs @@ -55,13 +55,7 @@ pub async fn group_summaries( group_id: team_id.to_string(), display_name: team_data.display_name.clone(), bio: team_data.bio.clone(), - avatar_url: util::route::team_avatar( - team_data - .profile_upload_id - .as_ref() - .map(common::Uuid::as_uuid), - team_data.profile_file_name.as_ref(), - ), + avatar_url: util::route::team_avatar(&team_data), external: models::GroupExternalLinks { profile: util::route::team_profile(team_id), chat: util::route::team_chat(team_id), diff --git a/svc/api/portal/src/convert.rs b/svc/api/portal/src/convert.rs index 19a13e0047..c3bf6f65af 100644 --- a/svc/api/portal/src/convert.rs +++ b/svc/api/portal/src/convert.rs @@ -10,16 +10,10 @@ pub fn game_summary( ) -> GlobalResult { Ok(models::GameSummary { game_id: internal_unwrap!(game.game_id).as_uuid().to_string(), - name_id: game.name_id, - display_name: game.display_name, - logo_url: util::route::game_logo( - game.logo_upload_id.as_ref().map(common::Uuid::as_uuid), - game.logo_file_name.as_ref(), - ), - banner_url: util::route::game_banner( - game.banner_upload_id.as_ref().map(common::Uuid::as_uuid), - game.banner_file_name.as_ref(), - ), + name_id: game.name_id.clone(), + display_name: game.display_name.clone(), + logo_url: util::route::game_logo(&game), + banner_url: util::route::game_banner(&game), url: game.url, developer: group_handle(team, true)?, @@ -36,10 +30,7 @@ pub fn group_handle( Ok(models::GroupHandle { group_id: team_id.to_string(), display_name: team.display_name.to_owned(), - avatar_url: util::route::team_avatar( - team.profile_upload_id.as_ref().map(common::Uuid::as_uuid), - team.profile_file_name.as_ref(), - ), + avatar_url: util::route::team_avatar(&team), external: models::GroupExternalLinks { profile: util::route::team_profile(team_id), chat: util::route::team_chat(team_id), diff --git a/svc/api/portal/src/route/games.rs b/svc/api/portal/src/route/games.rs index 03d4d14dd9..530f2e3ed3 100644 --- a/svc/api/portal/src/route/games.rs +++ b/svc/api/portal/src/route/games.rs @@ -41,14 +41,8 @@ pub async fn profile( game_id: game_id.to_string(), name_id: game.name_id.clone(), display_name: game.display_name.clone(), - logo_url: util::route::game_logo( - game.logo_upload_id.as_ref().map(common::Uuid::as_uuid), - game.logo_file_name.as_ref(), - ), - banner_url: util::route::game_banner( - game.banner_upload_id.as_ref().map(common::Uuid::as_uuid), - game.banner_file_name.as_ref(), - ), + logo_url: util::route::game_logo(&game), + banner_url: util::route::game_banner(&game), url: game.url.clone(), developer: developer_team.clone(), diff --git a/svc/api/route/Cargo.toml b/svc/api/route/Cargo.toml index de9453193a..7e5d209319 100644 --- a/svc/api/route/Cargo.toml +++ b/svc/api/route/Cargo.toml @@ -25,7 +25,7 @@ s3-util = { path = "../../../lib/s3-util" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } toml = "0.5" tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } diff --git a/svc/api/status/Cargo.toml b/svc/api/status/Cargo.toml index e936d1551f..c05ee0ae9c 100644 --- a/svc/api/status/Cargo.toml +++ b/svc/api/status/Cargo.toml @@ -30,7 +30,7 @@ s3-util = { path = "../../../lib/s3-util" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } url = "2.2.2" diff --git a/svc/api/status/src/auth.rs b/svc/api/status/src/auth.rs index c3ceb470f0..43d7568e6b 100644 --- a/svc/api/status/src/auth.rs +++ b/svc/api/status/src/auth.rs @@ -14,7 +14,6 @@ impl ApiAuth for Auth { ) -> GlobalResult { Self::rate_limit(rate_limit_ctx).await?; - // TODO: Don't hardcode this // TODO: Use JWT if let Some(api_token) = api_token { assert_eq_with!( @@ -23,9 +22,10 @@ impl ApiAuth for Auth { API_FORBIDDEN, reason = "Invalid auth" ); + Ok(Auth { _claims: None }) + } else { + internal_panic!("unreachable"); } - - Ok(Auth { _claims: None }) } async fn rate_limit(_rate_limit_ctx: AuthRateLimitCtx<'_>) -> GlobalResult<()> { diff --git a/svc/pkg/build/ops/create/Cargo.toml b/svc/pkg/build/ops/create/Cargo.toml index fbe7d31a5c..dd4c68f86b 100644 --- a/svc/pkg/build/ops/create/Cargo.toml +++ b/svc/pkg/build/ops/create/Cargo.toml @@ -14,7 +14,7 @@ game-get = { path = "../../../game/ops/get" } upload-prepare = { path = "../../../upload/ops/prepare" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/build/ops/create/src/lib.rs b/svc/pkg/build/ops/create/src/lib.rs index d2422e08fc..180548d67a 100644 --- a/svc/pkg/build/ops/create/src/lib.rs +++ b/svc/pkg/build/ops/create/src/lib.rs @@ -72,7 +72,7 @@ async fn handle( path: "image.tar".into(), mime: Some("application/x-tar".into()), content_length: image_file.content_length, - multipart: true, + multipart: ctx.multipart, ..Default::default() }, ], diff --git a/svc/pkg/build/ops/get/Cargo.toml b/svc/pkg/build/ops/get/Cargo.toml index 3fbdce1fea..8597e7384e 100644 --- a/svc/pkg/build/ops/get/Cargo.toml +++ b/svc/pkg/build/ops/get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/build/ops/list-for-game/Cargo.toml b/svc/pkg/build/ops/list-for-game/Cargo.toml index 3e8ec2ad1f..671466daad 100644 --- a/svc/pkg/build/ops/list-for-game/Cargo.toml +++ b/svc/pkg/build/ops/list-for-game/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/build/standalone/default-create/Cargo.toml b/svc/pkg/build/standalone/default-create/Cargo.toml index 86ef376ed3..8e8a55d70d 100644 --- a/svc/pkg/build/standalone/default-create/Cargo.toml +++ b/svc/pkg/build/standalone/default-create/Cargo.toml @@ -14,7 +14,7 @@ prost = "0.10" rivet-connection = { path = "../../../../../lib/connection" } reqwest = "0.11" rivet-pools = { path = "../../../../../lib/pools" } -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } uuid = { version = "1", features = ["v4"] } @@ -23,14 +23,13 @@ upload-complete = { path = "../../../upload/ops/complete" } upload-prepare = { path = "../../../upload/ops/prepare" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false features = [ - "runtime-tokio-native-tls", + "runtime-tokio", "postgres", "uuid", "json", - "tls", ] [dev-dependencies] diff --git a/svc/pkg/build/types/create.proto b/svc/pkg/build/types/create.proto index a186324dac..3ac96e19e3 100644 --- a/svc/pkg/build/types/create.proto +++ b/svc/pkg/build/types/create.proto @@ -10,6 +10,7 @@ message Request { string display_name = 2; optional string image_tag = 4; optional rivet.backend.upload.PrepareFile image_file = 3; + bool multipart = 6; // If provided, will look up the pre-uploaded upload ID for a given kind. optional string default_build_kind = 5; diff --git a/svc/pkg/captcha/ops/request/Cargo.toml b/svc/pkg/captcha/ops/request/Cargo.toml index dc4fdd8346..3c028a13ae 100644 --- a/svc/pkg/captcha/ops/request/Cargo.toml +++ b/svc/pkg/captcha/ops/request/Cargo.toml @@ -13,7 +13,7 @@ chrono = "0.4" util-captcha = { package = "rivet-util-captcha", path = "../../util" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-auth-user-remove/Cargo.toml b/svc/pkg/cdn/ops/namespace-auth-user-remove/Cargo.toml index 8af25eb0f8..f21760268f 100644 --- a/svc/pkg/cdn/ops/namespace-auth-user-remove/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-auth-user-remove/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-auth-user-update/Cargo.toml b/svc/pkg/cdn/ops/namespace-auth-user-update/Cargo.toml index 6defc8026e..cf89c5ca25 100644 --- a/svc/pkg/cdn/ops/namespace-auth-user-update/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-auth-user-update/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-create/Cargo.toml b/svc/pkg/cdn/ops/namespace-create/Cargo.toml index a95e157fdc..a5414d3cc6 100644 --- a/svc/pkg/cdn/ops/namespace-create/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-create/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-domain-create/Cargo.toml b/svc/pkg/cdn/ops/namespace-domain-create/Cargo.toml index 48d1ae2534..6a88d323d3 100644 --- a/svc/pkg/cdn/ops/namespace-domain-create/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-domain-create/Cargo.toml @@ -14,7 +14,7 @@ game-get = { path = "../../../game/ops/get" } game-resolve-namespace-id = { path = "../../../game/ops/resolve-namespace-id" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-domain-remove/Cargo.toml b/svc/pkg/cdn/ops/namespace-domain-remove/Cargo.toml index cb9cf7f996..73c2b52ea1 100644 --- a/svc/pkg/cdn/ops/namespace-domain-remove/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-domain-remove/Cargo.toml @@ -14,7 +14,7 @@ game-get = { path = "../../../game/ops/get" } game-resolve-namespace-id = { path = "../../../game/ops/resolve-namespace-id" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-get/Cargo.toml b/svc/pkg/cdn/ops/namespace-get/Cargo.toml index b5bb186bfa..9bd9b8eefb 100644 --- a/svc/pkg/cdn/ops/namespace-get/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/namespace-resolve-domain/Cargo.toml b/svc/pkg/cdn/ops/namespace-resolve-domain/Cargo.toml index 0708028a9b..9777553afd 100644 --- a/svc/pkg/cdn/ops/namespace-resolve-domain/Cargo.toml +++ b/svc/pkg/cdn/ops/namespace-resolve-domain/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/ns-auth-type-set/Cargo.toml b/svc/pkg/cdn/ops/ns-auth-type-set/Cargo.toml index 99577b1d5a..64374820ab 100644 --- a/svc/pkg/cdn/ops/ns-auth-type-set/Cargo.toml +++ b/svc/pkg/cdn/ops/ns-auth-type-set/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/ns-enable-domain-public-auth-set/Cargo.toml b/svc/pkg/cdn/ops/ns-enable-domain-public-auth-set/Cargo.toml index af4ae6edce..cf30bce172 100644 --- a/svc/pkg/cdn/ops/ns-enable-domain-public-auth-set/Cargo.toml +++ b/svc/pkg/cdn/ops/ns-enable-domain-public-auth-set/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/site-create/Cargo.toml b/svc/pkg/cdn/ops/site-create/Cargo.toml index deced8872b..70c503f842 100644 --- a/svc/pkg/cdn/ops/site-create/Cargo.toml +++ b/svc/pkg/cdn/ops/site-create/Cargo.toml @@ -14,7 +14,7 @@ game-get = { path = "../../../game/ops/get" } upload-prepare = { path = "../../../upload/ops/prepare" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/site-get/Cargo.toml b/svc/pkg/cdn/ops/site-get/Cargo.toml index 1e741f055f..32d72113a1 100644 --- a/svc/pkg/cdn/ops/site-get/Cargo.toml +++ b/svc/pkg/cdn/ops/site-get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/site-list-for-game/Cargo.toml b/svc/pkg/cdn/ops/site-list-for-game/Cargo.toml index c191f605cb..dbd26586ee 100644 --- a/svc/pkg/cdn/ops/site-list-for-game/Cargo.toml +++ b/svc/pkg/cdn/ops/site-list-for-game/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/version-get/Cargo.toml b/svc/pkg/cdn/ops/version-get/Cargo.toml index 5aea2f4077..ed7e13849e 100644 --- a/svc/pkg/cdn/ops/version-get/Cargo.toml +++ b/svc/pkg/cdn/ops/version-get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cdn/ops/version-publish/Cargo.toml b/svc/pkg/cdn/ops/version-publish/Cargo.toml index 827bee0f84..70535c0673 100644 --- a/svc/pkg/cdn/ops/version-publish/Cargo.toml +++ b/svc/pkg/cdn/ops/version-publish/Cargo.toml @@ -13,7 +13,7 @@ unzip-n = "0.1.2" itertools = "0.10.5" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cf-custom-hostname/ops/get/Cargo.toml b/svc/pkg/cf-custom-hostname/ops/get/Cargo.toml index 5a1bfdd049..186288f1ff 100644 --- a/svc/pkg/cf-custom-hostname/ops/get/Cargo.toml +++ b/svc/pkg/cf-custom-hostname/ops/get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cf-custom-hostname/ops/list-for-namespace-id/Cargo.toml b/svc/pkg/cf-custom-hostname/ops/list-for-namespace-id/Cargo.toml index 829ab4e10d..a41c3f82a1 100644 --- a/svc/pkg/cf-custom-hostname/ops/list-for-namespace-id/Cargo.toml +++ b/svc/pkg/cf-custom-hostname/ops/list-for-namespace-id/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cf-custom-hostname/ops/resolve-hostname/Cargo.toml b/svc/pkg/cf-custom-hostname/ops/resolve-hostname/Cargo.toml index a355f6a8d9..f7f7ef8f28 100644 --- a/svc/pkg/cf-custom-hostname/ops/resolve-hostname/Cargo.toml +++ b/svc/pkg/cf-custom-hostname/ops/resolve-hostname/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-message/ops/get/Cargo.toml b/svc/pkg/chat-message/ops/get/Cargo.toml index 2e07aea2ad..d1aefa9c6f 100644 --- a/svc/pkg/chat-message/ops/get/Cargo.toml +++ b/svc/pkg/chat-message/ops/get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-message/ops/list-for-user/Cargo.toml b/svc/pkg/chat-message/ops/list-for-user/Cargo.toml index 61e9bcbdd3..e3f6bbfa72 100644 --- a/svc/pkg/chat-message/ops/list-for-user/Cargo.toml +++ b/svc/pkg/chat-message/ops/list-for-user/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-message/ops/list/Cargo.toml b/svc/pkg/chat-message/ops/list/Cargo.toml index 9f0d52395e..4fe04e50f1 100644 --- a/svc/pkg/chat-message/ops/list/Cargo.toml +++ b/svc/pkg/chat-message/ops/list/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/ops/get-for-topic/Cargo.toml b/svc/pkg/chat-thread/ops/get-for-topic/Cargo.toml index 40a8fce677..7d3d4395a5 100644 --- a/svc/pkg/chat-thread/ops/get-for-topic/Cargo.toml +++ b/svc/pkg/chat-thread/ops/get-for-topic/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/ops/get/Cargo.toml b/svc/pkg/chat-thread/ops/get/Cargo.toml index afea653a48..02f309572f 100644 --- a/svc/pkg/chat-thread/ops/get/Cargo.toml +++ b/svc/pkg/chat-thread/ops/get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/ops/list-for-participant/Cargo.toml b/svc/pkg/chat-thread/ops/list-for-participant/Cargo.toml index ace6222fa9..7dca39cf89 100644 --- a/svc/pkg/chat-thread/ops/list-for-participant/Cargo.toml +++ b/svc/pkg/chat-thread/ops/list-for-participant/Cargo.toml @@ -14,7 +14,7 @@ prost = "0.10" user-team-list = { path = "../../../user/ops/team-list" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/ops/recent-for-user/Cargo.toml b/svc/pkg/chat-thread/ops/recent-for-user/Cargo.toml index db74712745..af7831fda5 100644 --- a/svc/pkg/chat-thread/ops/recent-for-user/Cargo.toml +++ b/svc/pkg/chat-thread/ops/recent-for-user/Cargo.toml @@ -17,7 +17,7 @@ chat-thread-get = { path = "../get" } chat-thread-list-for-participant = { path = "../list-for-participant" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/ops/unread-count/Cargo.toml b/svc/pkg/chat-thread/ops/unread-count/Cargo.toml index b75664defe..853f61db9d 100644 --- a/svc/pkg/chat-thread/ops/unread-count/Cargo.toml +++ b/svc/pkg/chat-thread/ops/unread-count/Cargo.toml @@ -15,7 +15,7 @@ serde_json = "1.0" chat-last-read-ts-get = { path = "../../../chat/ops/last-read-ts-get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat-thread/worker/Cargo.toml b/svc/pkg/chat-thread/worker/Cargo.toml index 0afb80b59b..ec40af319d 100644 --- a/svc/pkg/chat-thread/worker/Cargo.toml +++ b/svc/pkg/chat-thread/worker/Cargo.toml @@ -15,7 +15,7 @@ lazy_static = "1.4.0" chrono = "0.4" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/chat/ops/last-read-ts-get/Cargo.toml b/svc/pkg/chat/ops/last-read-ts-get/Cargo.toml index bc21da748b..d9aa23f091 100644 --- a/svc/pkg/chat/ops/last-read-ts-get/Cargo.toml +++ b/svc/pkg/chat/ops/last-read-ts-get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/game-config-create/Cargo.toml b/svc/pkg/cloud/ops/game-config-create/Cargo.toml index 3deb7aa33d..94af0569df 100644 --- a/svc/pkg/cloud/ops/game-config-create/Cargo.toml +++ b/svc/pkg/cloud/ops/game-config-create/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/game-config-get/Cargo.toml b/svc/pkg/cloud/ops/game-config-get/Cargo.toml index f710928b59..2fc404c910 100644 --- a/svc/pkg/cloud/ops/game-config-get/Cargo.toml +++ b/svc/pkg/cloud/ops/game-config-get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/game-token-create/Cargo.toml b/svc/pkg/cloud/ops/game-token-create/Cargo.toml index 2c9996c3e4..fb1c97a00c 100644 --- a/svc/pkg/cloud/ops/game-token-create/Cargo.toml +++ b/svc/pkg/cloud/ops/game-token-create/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/namespace-create/Cargo.toml b/svc/pkg/cloud/ops/namespace-create/Cargo.toml index 68b8f9e7fe..a4f75a47e8 100644 --- a/svc/pkg/cloud/ops/namespace-create/Cargo.toml +++ b/svc/pkg/cloud/ops/namespace-create/Cargo.toml @@ -18,7 +18,7 @@ kv-config-namespace-create = { path = "../../../kv-config/ops/namespace-create" identity-config-namespace-create = { path = "../../../identity-config/ops/namespace-create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/namespace-get/Cargo.toml b/svc/pkg/cloud/ops/namespace-get/Cargo.toml index 8cc109f520..9d3d60fcf8 100644 --- a/svc/pkg/cloud/ops/namespace-get/Cargo.toml +++ b/svc/pkg/cloud/ops/namespace-get/Cargo.toml @@ -16,7 +16,7 @@ kv-config-namespace-get = { path = "../../../kv-config/ops/namespace-get" } mm-config-namespace-get = { path = "../../../mm-config/ops/namespace-get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/namespace-token-development-create/Cargo.toml b/svc/pkg/cloud/ops/namespace-token-development-create/Cargo.toml index b935e592a5..16e8a2ae4e 100644 --- a/svc/pkg/cloud/ops/namespace-token-development-create/Cargo.toml +++ b/svc/pkg/cloud/ops/namespace-token-development-create/Cargo.toml @@ -15,7 +15,7 @@ game-token-development-validate = { path = "../../../game/ops/token-development- token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/namespace-token-public-create/Cargo.toml b/svc/pkg/cloud/ops/namespace-token-public-create/Cargo.toml index 3d6563061f..2ba6eb987b 100644 --- a/svc/pkg/cloud/ops/namespace-token-public-create/Cargo.toml +++ b/svc/pkg/cloud/ops/namespace-token-public-create/Cargo.toml @@ -15,7 +15,7 @@ game-namespace-get = { path = "../../../game/ops/namespace-get" } token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/version-get/Cargo.toml b/svc/pkg/cloud/ops/version-get/Cargo.toml index f6d4ce3cc5..e22fc993c6 100644 --- a/svc/pkg/cloud/ops/version-get/Cargo.toml +++ b/svc/pkg/cloud/ops/version-get/Cargo.toml @@ -18,7 +18,7 @@ mm-config-version-get = { path = "../../../mm-config/ops/version-get" } module-game-version-get = { path = "../../../module/ops/game-version-get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/cloud/ops/version-publish/Cargo.toml b/svc/pkg/cloud/ops/version-publish/Cargo.toml index 47cbc69a45..43da7e59e4 100644 --- a/svc/pkg/cloud/ops/version-publish/Cargo.toml +++ b/svc/pkg/cloud/ops/version-publish/Cargo.toml @@ -25,7 +25,7 @@ module-game-version-prepare = { path = "../../../module/ops/game-version-prepare module-game-version-publish = { path = "../../../module/ops/game-version-publish" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/custom-user-avatar/ops/list-for-game/Cargo.toml b/svc/pkg/custom-user-avatar/ops/list-for-game/Cargo.toml index 24e87ab191..4fce1dc75b 100644 --- a/svc/pkg/custom-user-avatar/ops/list-for-game/Cargo.toml +++ b/svc/pkg/custom-user-avatar/ops/list-for-game/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/custom-user-avatar/ops/upload-complete/Cargo.toml b/svc/pkg/custom-user-avatar/ops/upload-complete/Cargo.toml index 4f85ad4e95..24abf1e7b6 100644 --- a/svc/pkg/custom-user-avatar/ops/upload-complete/Cargo.toml +++ b/svc/pkg/custom-user-avatar/ops/upload-complete/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" upload-complete = { path = "../../../upload/ops/complete" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/debug/ops/email-res/Cargo.toml b/svc/pkg/debug/ops/email-res/Cargo.toml index 2e32780ca8..009dc7a79a 100644 --- a/svc/pkg/debug/ops/email-res/Cargo.toml +++ b/svc/pkg/debug/ops/email-res/Cargo.toml @@ -13,7 +13,7 @@ rand = "0.8" serde_json = "1.0" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/email-verification/ops/complete/Cargo.toml b/svc/pkg/email-verification/ops/complete/Cargo.toml index aff5e415de..0dcf81bd18 100644 --- a/svc/pkg/email-verification/ops/complete/Cargo.toml +++ b/svc/pkg/email-verification/ops/complete/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/email-verification/ops/create/Cargo.toml b/svc/pkg/email-verification/ops/create/Cargo.toml index 5a0847998f..46bfea86ad 100644 --- a/svc/pkg/email-verification/ops/create/Cargo.toml +++ b/svc/pkg/email-verification/ops/create/Cargo.toml @@ -18,7 +18,7 @@ email-send = { path = "../../../email/ops/send" } game-get = { path = "../../../game/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/email-verification/ops/create/src/lib.rs b/svc/pkg/email-verification/ops/create/src/lib.rs index 368e2c6bbf..0cf8d43568 100644 --- a/svc/pkg/email-verification/ops/create/src/lib.rs +++ b/svc/pkg/email-verification/ops/create/src/lib.rs @@ -52,9 +52,7 @@ async fn handle( json!({ "game_display_name": game.display_name.clone(), "game_logo_url": util::route::game_logo( - game.logo_upload_id.as_ref().map(common::Uuid::as_uuid), - game.logo_file_name.as_ref(), - ), + &game), "verification_code": code, }), ) diff --git a/svc/pkg/faker/ops/game-version/src/lib.rs b/svc/pkg/faker/ops/game-version/src/lib.rs index 036d2dca40..c6eafe97d7 100644 --- a/svc/pkg/faker/ops/game-version/src/lib.rs +++ b/svc/pkg/faker/ops/game-version/src/lib.rs @@ -12,67 +12,68 @@ async fn handle( }) .await?; - let config = if let Some(config) = ctx.override_config.clone() { - config - } else { - backend::cloud::VersionConfig { - cdn: if let Some(config) = ctx.override_cdn_config.clone() { - config.config - } else { - let cdn_site_res = op!([ctx] faker_cdn_site { - game_id: Some(*game_id), - }) - .await?; - let site_id = internal_unwrap!(cdn_site_res.site_id); + let config = + if let Some(config) = ctx.override_config.clone() { + config + } else { + backend::cloud::VersionConfig { + cdn: if let Some(config) = ctx.override_cdn_config.clone() { + config.config + } else { + let cdn_site_res = op!([ctx] faker_cdn_site { + game_id: Some(*game_id), + }) + .await?; + let site_id = internal_unwrap!(cdn_site_res.site_id); - Some(VersionConfig { - site_id: Some(*site_id), - routes: vec![ - Route { - glob: Some(util::glob::Glob::parse("test-glob")?.into()), - priority: 0, - middlewares: vec![Middleware { - kind: Some(middleware::Kind::CustomHeaders( - CustomHeadersMiddleware { - headers: vec![custom_headers_middleware::Header { - name: "header-name".to_string(), - value: "header-value".to_string(), - }], - }, - )), - }], - }, - Route { - glob: Some(util::glob::Glob::parse("test-glob2")?.into()), - priority: 1, - middlewares: vec![Middleware { - kind: Some(middleware::Kind::CustomHeaders( - CustomHeadersMiddleware { - headers: vec![custom_headers_middleware::Header { - name: "header-name2".to_string(), - value: "header-value2".to_string(), - }], - }, - )), - }], - }, - ], - }) - }, - matchmaker: if let Some(config) = ctx.override_mm_config.clone() { - config.config - } else { - let build_res = op!([ctx] faker_build { - game_id: Some(*game_id), - image: faker::build::Image::MmLobbyAutoReady as i32, - }) - .await?; - let build_id = internal_unwrap!(build_res.build_id); + Some(VersionConfig { + site_id: Some(*site_id), + routes: vec![ + Route { + glob: Some(util::glob::Glob::parse("test-glob")?.into()), + priority: 0, + middlewares: vec![Middleware { + kind: Some(middleware::Kind::CustomHeaders( + CustomHeadersMiddleware { + headers: vec![custom_headers_middleware::Header { + name: "header-name".to_string(), + value: "header-value".to_string(), + }], + }, + )), + }], + }, + Route { + glob: Some(util::glob::Glob::parse("test-glob2")?.into()), + priority: 1, + middlewares: vec![Middleware { + kind: Some(middleware::Kind::CustomHeaders( + CustomHeadersMiddleware { + headers: vec![custom_headers_middleware::Header { + name: "header-name2".to_string(), + value: "header-value2".to_string(), + }], + }, + )), + }], + }, + ], + }) + }, + matchmaker: if let Some(config) = ctx.override_mm_config.clone() { + config.config + } else { + let build_res = op!([ctx] faker_build { + game_id: Some(*game_id), + image: faker::build::Image::MmLobbyAutoReady as i32, + }) + .await?; + let build_id = internal_unwrap!(build_res.build_id); - Some(backend::matchmaker::VersionConfig { - lobby_groups: ctx.override_lobby_groups.clone().map_or_else( - || { - vec![backend::matchmaker::LobbyGroup { + Some(backend::matchmaker::VersionConfig { + lobby_groups: ctx.override_lobby_groups.clone().map_or_else( + || { + vec![backend::matchmaker::LobbyGroup { name_id: "test-1".into(), regions: region_list_res @@ -82,7 +83,13 @@ async fn handle( .map(|region_id| backend::matchmaker::lobby_group::Region { region_id: Some(region_id), tier_name_id: util_mm::test::TIER_NAME_ID.to_owned(), - idle_lobbies: None, + idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies { + min_idle_lobbies: 0, + // Set a high max lobby count in case this is + // coming from a test that test mm-lobby-create + // without creating an associated player + max_idle_lobbies: 32, + }), }) .collect(), @@ -103,39 +110,39 @@ async fn handle( .into(), ), }] - }, - |v| v.lobby_groups, - ), - captcha: ctx - .override_captcha - .clone() - .map_or_else(|| None, |config| config.captcha_config), - }) - }, - kv: if let Some(config) = ctx.override_kv_config.clone() { - config.config - } else { - Some(backend::kv::VersionConfig {}) - }, - identity: if let Some(config) = ctx.override_identity_config.clone() { - config.config - } else { - Some(backend::identity::VersionConfig { - custom_display_names: vec![backend::identity::CustomDisplayName { - display_name: "Guest".to_string(), - }], - custom_avatars: Vec::new(), - }) - }, - module: if let Some(config) = ctx.override_module_config.clone() { - config.config - } else { - Some(backend::module::GameVersionConfig { - dependencies: Vec::new(), - }) - }, - } - }; + }, + |v| v.lobby_groups, + ), + captcha: ctx + .override_captcha + .clone() + .map_or_else(|| None, |config| config.captcha_config), + }) + }, + kv: if let Some(config) = ctx.override_kv_config.clone() { + config.config + } else { + Some(backend::kv::VersionConfig {}) + }, + identity: if let Some(config) = ctx.override_identity_config.clone() { + config.config + } else { + Some(backend::identity::VersionConfig { + custom_display_names: vec![backend::identity::CustomDisplayName { + display_name: "Guest".to_string(), + }], + custom_avatars: Vec::new(), + }) + }, + module: if let Some(config) = ctx.override_module_config.clone() { + config.config + } else { + Some(backend::module::GameVersionConfig { + dependencies: Vec::new(), + }) + }, + } + }; let version_create_res = op!([ctx] cloud_version_publish { game_id: Some(*game_id), diff --git a/svc/pkg/faker/ops/mm-lobby/src/lib.rs b/svc/pkg/faker/ops/mm-lobby/src/lib.rs index 5f66a919a0..480abb3b1d 100644 --- a/svc/pkg/faker/ops/mm-lobby/src/lib.rs +++ b/svc/pkg/faker/ops/mm-lobby/src/lib.rs @@ -70,7 +70,13 @@ async fn handle( regions: vec![backend::matchmaker::lobby_group::Region { region_id: region_res.region_id, tier_name_id: util_mm::test::TIER_NAME_ID.to_owned(), - idle_lobbies: None, + idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies { + min_idle_lobbies: 0, + // Set a high max lobby count in case this is + // coming from a test that test mm-lobby-create + // without creating an associated player + max_idle_lobbies: 32, + }), }], max_players_normal, max_players_direct, diff --git a/svc/pkg/faker/ops/region/Cargo.toml b/svc/pkg/faker/ops/region/Cargo.toml index 51d1ad74e2..8d867ec66c 100644 --- a/svc/pkg/faker/ops/region/Cargo.toml +++ b/svc/pkg/faker/ops/region/Cargo.toml @@ -14,7 +14,7 @@ region-get = { path = "../../../region/ops/get" } region-list = { path = "../../../region/ops/list" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/create/Cargo.toml b/svc/pkg/game-user/ops/create/Cargo.toml index 1bc747b181..6f51cbe6e9 100644 --- a/svc/pkg/game-user/ops/create/Cargo.toml +++ b/svc/pkg/game-user/ops/create/Cargo.toml @@ -15,7 +15,7 @@ util-game-user = { package = "rivet-util-game-user", path = "../../util" } token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/get/Cargo.toml b/svc/pkg/game-user/ops/get/Cargo.toml index 7caeebcac5..b9e18a51f1 100644 --- a/svc/pkg/game-user/ops/get/Cargo.toml +++ b/svc/pkg/game-user/ops/get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/link-create/Cargo.toml b/svc/pkg/game-user/ops/link-create/Cargo.toml index 2641a89814..6dc921d1bd 100644 --- a/svc/pkg/game-user/ops/link-create/Cargo.toml +++ b/svc/pkg/game-user/ops/link-create/Cargo.toml @@ -15,7 +15,7 @@ game-user-get = { path = "../get" } token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/link-get/Cargo.toml b/svc/pkg/game-user/ops/link-get/Cargo.toml index 27f3cac553..ac06ab5f1a 100644 --- a/svc/pkg/game-user/ops/link-get/Cargo.toml +++ b/svc/pkg/game-user/ops/link-get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/list-for-user/Cargo.toml b/svc/pkg/game-user/ops/list-for-user/Cargo.toml index f517494567..091a0886ee 100644 --- a/svc/pkg/game-user/ops/list-for-user/Cargo.toml +++ b/svc/pkg/game-user/ops/list-for-user/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/ops/recent-session-list/Cargo.toml b/svc/pkg/game-user/ops/recent-session-list/Cargo.toml index 7f43946381..ac365db3b5 100644 --- a/svc/pkg/game-user/ops/recent-session-list/Cargo.toml +++ b/svc/pkg/game-user/ops/recent-session-list/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game-user/worker/Cargo.toml b/svc/pkg/game-user/worker/Cargo.toml index 7bb8d102a3..58c0914233 100644 --- a/svc/pkg/game-user/worker/Cargo.toml +++ b/svc/pkg/game-user/worker/Cargo.toml @@ -19,7 +19,7 @@ token-exchange = { path = "../../token/ops/exchange" } token-revoke = { path = "../../token/ops/revoke" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game/ops/banner-upload-complete/Cargo.toml b/svc/pkg/game/ops/banner-upload-complete/Cargo.toml index 810390d9db..59078fd7c5 100644 --- a/svc/pkg/game/ops/banner-upload-complete/Cargo.toml +++ b/svc/pkg/game/ops/banner-upload-complete/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" upload-complete = { path = "../../../upload/ops/complete" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game/ops/create/Cargo.toml b/svc/pkg/game/ops/create/Cargo.toml index 3bd77072fa..e9560bd588 100644 --- a/svc/pkg/game/ops/create/Cargo.toml +++ b/svc/pkg/game/ops/create/Cargo.toml @@ -14,7 +14,7 @@ game-validate = { path = "../validate" } team-dev-get = { path = "../../../team-dev/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game/ops/get/Cargo.toml b/svc/pkg/game/ops/get/Cargo.toml index 95c61992f2..9411390013 100644 --- a/svc/pkg/game/ops/get/Cargo.toml +++ b/svc/pkg/game/ops/get/Cargo.toml @@ -14,7 +14,7 @@ upload-file-list = { path = "../../../upload/ops/file-list" } upload-get = { path = "../../../upload/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/game/ops/get/src/lib.rs b/svc/pkg/game/ops/get/src/lib.rs index 08209ded0d..5d7932e8f2 100644 --- a/svc/pkg/game/ops/get/src/lib.rs +++ b/svc/pkg/game/ops/get/src/lib.rs @@ -79,7 +79,7 @@ async fn handle(ctx: OperationContext) -> GlobalResult::into); // Fetch all information relating to the logo image - let (logo_upload_complete_ts, logo_file_name) = { + let (logo_upload_complete_ts, logo_file_name, logo_provider) = { let upload = upload_res .uploads .iter() @@ -95,14 +95,14 @@ async fn handle(ctx: OperationContext) -> GlobalResult) -> GlobalResult) -> GlobalResult s3_util::Provider::Minio, + backend::upload::Provider::Backblaze => s3_util::Provider::Backblaze, + backend::upload::Provider::Aws => s3_util::Provider::Aws, + }; + match DELIVERY_METHOD { DeliveryMethod::Ats => { - // TODO: Auto-generate password for this & encode in to infra/salt/salt/traffic_server/files/consul/traffic_server.hcl.j2 - // See tf/svc/traefik/main.tf - // let http_auth = "job_run_image:XXXX"; - let http_auth: String = todo!("need to auto-generate password"); - - // TODO: Unproxied storage endopint was removed. Replace with Nebula address. We can't - // use Consul to resolve this though, since Consul is not installed on the edge nodes. - // Pull the image from the CDN region instead of the default region for - // faster boot times. + let ats_url = if cdn_region.name_id == "lnd-atl" { + "10.0.47.2" + } + else { + tracing::info!(?cdn_region.name_id); + internal_panic!("invalid cdn region"); + }; + let upload_id = internal_unwrap!(upload.upload_id).as_uuid(); let addr = format!( - "https://{auth}@cdn.{cdn_region}.{domain}/build/{upload_id}/image.tar", - cdn_region = cdn_region.name_id, - auth = http_auth, - domain = util::env::domain_main(), + "http://{ats_url}:9300/s3-cache/{provider}/{namespace}-bucket-build/{upload_id}/image.tar", + ats_url = ats_url, + provider = heck::KebabCase::to_kebab_case(provider.to_string().as_str()), + namespace = util::env::namespace(), upload_id = upload_id, ); @@ -755,8 +765,8 @@ async fn resolve_image_artifact_url( // Build client let s3_client = s3_util::Client::from_env_opt( - &bucket, - s3_util::Provider::default()?, + bucket, + provider, s3_util::EndpointKind::InternalResolved, ) .await?; diff --git a/svc/pkg/mm/worker/src/workers/lobby_create/nomad_job.rs b/svc/pkg/mm/worker/src/workers/lobby_create/nomad_job.rs index 4452791ebb..08d031a5cc 100644 --- a/svc/pkg/mm/worker/src/workers/lobby_create/nomad_job.rs +++ b/svc/pkg/mm/worker/src/workers/lobby_create/nomad_job.rs @@ -122,7 +122,6 @@ pub fn gen_lobby_docker_job( ("RIVET_IDENTITY_API_URL", "api-identity"), ("RIVET_KV_API_URL", "api-kv"), ("RIVET_MATCHMAKER_API_URL", "api-matchmaker"), - ("RIVET_PARTY_API_URL", "api-party"), ] .iter() .map(|(env, service)| (env.to_string(), util::env::svc_router_url(service))), diff --git a/svc/pkg/mm/worker/src/workers/lobby_find/mod.rs b/svc/pkg/mm/worker/src/workers/lobby_find/mod.rs index 4ca64b9ba0..8734cc3635 100644 --- a/svc/pkg/mm/worker/src/workers/lobby_find/mod.rs +++ b/svc/pkg/mm/worker/src/workers/lobby_find/mod.rs @@ -537,7 +537,7 @@ async fn insert_to_crdb( .bind(lobby_group_config.lobby_group.max_players_normal as i64) .bind(lobby_group_config.lobby_group.max_players_direct as i64) .bind(lobby_group_config.lobby_group.max_players_party as i64) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } @@ -561,7 +561,7 @@ async fn insert_to_crdb( .bind(lobby_id) .bind(auto_create_lobby) .bind(util_mm::FindQueryStatus::Pending as i64) - .execute(&mut *tx) + .execute(&mut **tx) .await?; // Insert players @@ -592,7 +592,7 @@ async fn insert_to_crdb( ) .bind(now_ts) .bind(ray_id) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } diff --git a/svc/pkg/mm/worker/tests/lobby_create.rs b/svc/pkg/mm/worker/tests/lobby_create.rs index c7a7435a66..1b28fbce5a 100644 --- a/svc/pkg/mm/worker/tests/lobby_create.rs +++ b/svc/pkg/mm/worker/tests/lobby_create.rs @@ -34,7 +34,11 @@ async fn setup(ctx: &TestCtx) -> Setup { regions: vec![backend::matchmaker::lobby_group::Region { region_id: Some(region_id.into()), tier_name_id: util_mm::test::TIER_NAME_ID.to_owned(), - idle_lobbies: None, + idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies { + min_idle_lobbies: 0, + // Don't auto-destory lobbies from tests + max_idle_lobbies: 32, + }), }], max_players_normal: 8, max_players_direct: 10, @@ -87,7 +91,11 @@ async fn setup(ctx: &TestCtx) -> Setup { regions: vec![backend::matchmaker::lobby_group::Region { region_id: Some(region_id.into()), tier_name_id: util_mm::test::TIER_NAME_ID.to_owned(), - idle_lobbies: None, + idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies { + min_idle_lobbies: 0, + // See above + max_idle_lobbies: 32, + }), }], max_players_normal: 8, max_players_direct: 10, diff --git a/svc/pkg/module/ops/game-version-get/Cargo.toml b/svc/pkg/module/ops/game-version-get/Cargo.toml index 75c4b96400..d34bcb0810 100644 --- a/svc/pkg/module/ops/game-version-get/Cargo.toml +++ b/svc/pkg/module/ops/game-version-get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/game-version-publish/Cargo.toml b/svc/pkg/module/ops/game-version-publish/Cargo.toml index f66c938c6f..6247971b6a 100644 --- a/svc/pkg/module/ops/game-version-publish/Cargo.toml +++ b/svc/pkg/module/ops/game-version-publish/Cargo.toml @@ -13,7 +13,7 @@ unzip-n = "0.1.2" itertools = "0.10.5" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/get/Cargo.toml b/svc/pkg/module/ops/get/Cargo.toml index a4b31a8bf2..411fc6a8cf 100644 --- a/svc/pkg/module/ops/get/Cargo.toml +++ b/svc/pkg/module/ops/get/Cargo.toml @@ -11,7 +11,7 @@ prost = "0.10" rivet-operation = { path = "../../../../../lib/operation/core" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/instance-call/Cargo.toml b/svc/pkg/module/ops/instance-call/Cargo.toml index edb84d378c..f517394032 100644 --- a/svc/pkg/module/ops/instance-call/Cargo.toml +++ b/svc/pkg/module/ops/instance-call/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1.0", features = ["derive"] } reqwest = { version = "0.11", features = ["json"] } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/instance-get/Cargo.toml b/svc/pkg/module/ops/instance-get/Cargo.toml index 2ab74c9491..25cce92fb6 100644 --- a/svc/pkg/module/ops/instance-get/Cargo.toml +++ b/svc/pkg/module/ops/instance-get/Cargo.toml @@ -11,7 +11,7 @@ prost = "0.10" rivet-operation = { path = "../../../../../lib/operation/core" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/ns-instance-get/Cargo.toml b/svc/pkg/module/ops/ns-instance-get/Cargo.toml index 0cb0e13fc0..8b08930c34 100644 --- a/svc/pkg/module/ops/ns-instance-get/Cargo.toml +++ b/svc/pkg/module/ops/ns-instance-get/Cargo.toml @@ -11,7 +11,7 @@ prost = "0.10" rivet-operation = { path = "../../../../../lib/operation/core" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/ops/version-get/Cargo.toml b/svc/pkg/module/ops/version-get/Cargo.toml index e1c9fc8613..3e208ee6d3 100644 --- a/svc/pkg/module/ops/version-get/Cargo.toml +++ b/svc/pkg/module/ops/version-get/Cargo.toml @@ -13,7 +13,7 @@ rivet-operation = { path = "../../../../../lib/operation/core" } module-get = { path = "../get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/module/worker/Cargo.toml b/svc/pkg/module/worker/Cargo.toml index 500082973a..7838139ff7 100644 --- a/svc/pkg/module/worker/Cargo.toml +++ b/svc/pkg/module/worker/Cargo.toml @@ -22,15 +22,14 @@ module-instance-get = { path = "../ops/instance-get" } module-game-version-get = { path = "../ops/game-version-get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false features = [ - "runtime-tokio-native-tls", + "runtime-tokio", "postgres", "macros", "uuid", "json", - "tls", ] [dev-dependencies] diff --git a/svc/pkg/module/worker/Service.toml b/svc/pkg/module/worker/Service.toml index 67e38c5305..36e10767e5 100644 --- a/svc/pkg/module/worker/Service.toml +++ b/svc/pkg/module/worker/Service.toml @@ -7,4 +7,4 @@ kind = "rust" [consumer] [secrets] -"fly/auth_token" = {} +"fly/auth_token" = { optional = true } diff --git a/svc/pkg/module/worker/src/workers/instance_create.rs b/svc/pkg/module/worker/src/workers/instance_create.rs index db3968a5f9..62cbe8613c 100644 --- a/svc/pkg/module/worker/src/workers/instance_create.rs +++ b/svc/pkg/module/worker/src/workers/instance_create.rs @@ -196,7 +196,7 @@ async fn insert_instance( .bind(instance_id) .bind(version_id) .bind(now) - .execute(&mut *tx) + .execute(&mut **tx) .await?; match internal_unwrap!(msg.driver) { @@ -208,7 +208,7 @@ async fn insert_instance( " )) .bind(instance_id) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } module::msg::instance_create::message::Driver::Fly(_) => { @@ -219,7 +219,7 @@ async fn insert_instance( " )) .bind(instance_id) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } } diff --git a/svc/pkg/module/worker/src/workers/ns_version_set.rs b/svc/pkg/module/worker/src/workers/ns_version_set.rs index 30f14c5e79..f284fc485a 100644 --- a/svc/pkg/module/worker/src/workers/ns_version_set.rs +++ b/svc/pkg/module/worker/src/workers/ns_version_set.rs @@ -33,10 +33,13 @@ async fn worker( version_ids: vec![version_id.into()], }) .await?; - let game_version = internal_unwrap_owned!(game_versions + let Some(game_version) = game_versions .versions .first() - .and_then(|x| x.config.as_ref())); + .and_then(|x| x.config.as_ref()) else { + tracing::info!("no game version found"); + return Ok(()); + }; let new_version_keys = game_version .dependencies diff --git a/svc/pkg/module/worker/src/workers/version_create.rs b/svc/pkg/module/worker/src/workers/version_create.rs index d6d2423cef..31b959b4a3 100644 --- a/svc/pkg/module/worker/src/workers/version_create.rs +++ b/svc/pkg/module/worker/src/workers/version_create.rs @@ -61,7 +61,7 @@ async fn update_db( .bind(TryInto::::try_into(msg.major)?) .bind(TryInto::::try_into(msg.minor)?) .bind(TryInto::::try_into(msg.patch)?) - .execute(&mut *tx) + .execute(&mut **tx) .await?; match internal_unwrap!(msg.image) { @@ -74,7 +74,7 @@ async fn update_db( )) .bind(version_id) .bind(&docker.image_tag) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } } @@ -90,7 +90,7 @@ async fn update_db( .bind(&script.name) .bind(&script.request_schema) .bind(&script.response_schema) - .execute(&mut *tx) + .execute(&mut **tx) .await?; if script.callable.is_some() { @@ -102,7 +102,7 @@ async fn update_db( )) .bind(version_id) .bind(&script.name) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } } diff --git a/svc/pkg/nomad-log/standalone/follower/Cargo.toml b/svc/pkg/nomad-log/standalone/follower/Cargo.toml index 8b1f47e06e..f5db65e106 100644 --- a/svc/pkg/nomad-log/standalone/follower/Cargo.toml +++ b/svc/pkg/nomad-log/standalone/follower/Cargo.toml @@ -22,7 +22,7 @@ rivet-pools = { path = "../../../../../lib/pools" } rivet-runtime = { path = "../../../../../lib/runtime" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } util-job = { package = "rivet-util-job", path = "../../../job/util" } diff --git a/svc/pkg/push-notification/worker/src/workers/send.rs b/svc/pkg/push-notification/worker/src/workers/send.rs index 29558df3fb..c4c5aefa58 100644 --- a/svc/pkg/push-notification/worker/src/workers/send.rs +++ b/svc/pkg/push-notification/worker/src/workers/send.rs @@ -22,7 +22,7 @@ async fn worker( )); match service { - // Send push notificaton through Firebase + // Send push notification through Firebase backend::notification::NotificationService::Firebase => { let row = sqlx::query_as::<_, (Option,)>(indoc!( " @@ -63,11 +63,7 @@ async fn worker( &text.body.chars().collect::>(), 1024, )?; - let icon = util::route::user_avatar( - &user.avatar_id, - user.profile_upload_id.as_ref().map(common::Uuid::as_uuid), - user.profile_file_name.as_ref(), - ); + let icon = util::route::user_avatar(&user); let click_url = format!("/threads/{}", thread_id); let mut notif_builder = NotificationBuilder::new(); diff --git a/svc/pkg/region/ops/get/Cargo.toml b/svc/pkg/region/ops/get/Cargo.toml index b49c30c467..40db38d8a9 100644 --- a/svc/pkg/region/ops/get/Cargo.toml +++ b/svc/pkg/region/ops/get/Cargo.toml @@ -12,7 +12,7 @@ prost = "0.10" util-region = { package = "rivet-util-region", path = "../../util" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/region/ops/list/Cargo.toml b/svc/pkg/region/ops/list/Cargo.toml index 7832a3cfc2..bced1f244c 100644 --- a/svc/pkg/region/ops/list/Cargo.toml +++ b/svc/pkg/region/ops/list/Cargo.toml @@ -12,7 +12,7 @@ prost = "0.10" util-region = { package = "rivet-util-region", path = "../../util" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/region/ops/recommend/Cargo.toml b/svc/pkg/region/ops/recommend/Cargo.toml index f5206d92d1..4e404562a4 100644 --- a/svc/pkg/region/ops/recommend/Cargo.toml +++ b/svc/pkg/region/ops/recommend/Cargo.toml @@ -15,7 +15,7 @@ ip-info = { path = "../../../ip/ops/info" } region-get = { path = "../get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/region/ops/resolve/Cargo.toml b/svc/pkg/region/ops/resolve/Cargo.toml index 3a88e14d30..fd5ad3ed92 100644 --- a/svc/pkg/region/ops/resolve/Cargo.toml +++ b/svc/pkg/region/ops/resolve/Cargo.toml @@ -12,7 +12,7 @@ prost = "0.10" util-region = { package = "rivet-util-region", path = "../../util" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/region/util/Cargo.toml b/svc/pkg/region/util/Cargo.toml index e2e4ed07f7..fbf58d6e40 100644 --- a/svc/pkg/region/util/Cargo.toml +++ b/svc/pkg/region/util/Cargo.toml @@ -8,5 +8,5 @@ license = "Apache-2.0" [dependencies] serde = { version = "1.0", features = ["derive"] } uuid = { version = "1", features = ["v4", "serde"] } -tokio = { version = "1.21", features = ["sync"] } +tokio = { version = "1.29", features = ["sync"] } serde_json = "1.0" diff --git a/svc/pkg/team-dev/ops/game-list/Cargo.toml b/svc/pkg/team-dev/ops/game-list/Cargo.toml index 1cf6674d22..d7befcd43f 100644 --- a/svc/pkg/team-dev/ops/game-list/Cargo.toml +++ b/svc/pkg/team-dev/ops/game-list/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team-dev/ops/get/Cargo.toml b/svc/pkg/team-dev/ops/get/Cargo.toml index 7fb6fecc06..ab92b162b0 100644 --- a/svc/pkg/team-dev/ops/get/Cargo.toml +++ b/svc/pkg/team-dev/ops/get/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team-dev/ops/halt/Cargo.toml b/svc/pkg/team-dev/ops/halt/Cargo.toml index 96ab475820..cad1eefa2c 100644 --- a/svc/pkg/team-dev/ops/halt/Cargo.toml +++ b/svc/pkg/team-dev/ops/halt/Cargo.toml @@ -14,7 +14,7 @@ mm-lobby-list-for-namespace = { path = "../../../mm/ops/lobby-list-for-namespace game-namespace-list = { path = "../../../game/ops/namespace-list" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team-dev/standalone/halt-collect/Cargo.toml b/svc/pkg/team-dev/standalone/halt-collect/Cargo.toml index 413c74f026..6b69fac489 100644 --- a/svc/pkg/team-dev/standalone/halt-collect/Cargo.toml +++ b/svc/pkg/team-dev/standalone/halt-collect/Cargo.toml @@ -16,21 +16,19 @@ rivet-connection = { path = "../../../../../lib/connection" } chrono = "0.4" rivet-pools = { path = "../../../../../lib/pools" } rivet-runtime = { path = "../../../../../lib/runtime" } -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } team-dev-halt = { path = "../../ops/halt" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false features = [ - "runtime-tokio-native-tls", "postgres", "uuid", "json", - "tls", "macros", ] diff --git a/svc/pkg/team-invite/ops/get/Cargo.toml b/svc/pkg/team-invite/ops/get/Cargo.toml index 0db1fb3b23..447c6629b9 100644 --- a/svc/pkg/team-invite/ops/get/Cargo.toml +++ b/svc/pkg/team-invite/ops/get/Cargo.toml @@ -12,7 +12,7 @@ chrono = "0.4" prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team-invite/worker/Cargo.toml b/svc/pkg/team-invite/worker/Cargo.toml index 948060affe..ef4ba13ffa 100644 --- a/svc/pkg/team-invite/worker/Cargo.toml +++ b/svc/pkg/team-invite/worker/Cargo.toml @@ -17,7 +17,7 @@ team-member-list = { path = "../../team/ops/member-list" } team-user-ban-get = { path = "../../team/ops/user-ban-get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team-invite/worker/src/workers/consume.rs b/svc/pkg/team-invite/worker/src/workers/consume.rs index 7c7acb5f9e..04ddba8a9c 100644 --- a/svc/pkg/team-invite/worker/src/workers/consume.rs +++ b/svc/pkg/team-invite/worker/src/workers/consume.rs @@ -111,7 +111,7 @@ async fn update_db( ", ) .bind(&code) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await?; let invitation_row = if let Some(invitation) = invitation_row { tracing::info!(?invitation, "found invitation"); @@ -188,13 +188,13 @@ async fn update_db( // Insert consumption sqlx::query("UPDATE invitations SET use_counter = use_counter + 1 WHERE code = $1") .bind(&code) - .execute(&mut *tx) + .execute(&mut **tx) .await?; sqlx::query("INSERT INTO invitation_uses (code, user_id, create_ts) VALUES ($1, $2, $3)") .bind(&code) .bind(user_id) .bind(now) - .execute(&mut *tx) + .execute(&mut **tx) .await?; Ok(DbOutput::Success { invitation_row }) diff --git a/svc/pkg/team/ops/avatar-upload-complete/Cargo.toml b/svc/pkg/team/ops/avatar-upload-complete/Cargo.toml index 64f50d8359..e7f69ccbb7 100644 --- a/svc/pkg/team/ops/avatar-upload-complete/Cargo.toml +++ b/svc/pkg/team/ops/avatar-upload-complete/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" upload-complete = { path = "../../../upload/ops/complete" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team/ops/get/Cargo.toml b/svc/pkg/team/ops/get/Cargo.toml index a01438c969..6f297432cc 100644 --- a/svc/pkg/team/ops/get/Cargo.toml +++ b/svc/pkg/team/ops/get/Cargo.toml @@ -14,7 +14,7 @@ upload-file-list = { path = "../../../upload/ops/file-list" } upload-get = { path = "../../../upload/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/team/ops/get/src/lib.rs b/svc/pkg/team/ops/get/src/lib.rs index 44be3980a8..d200677cbb 100644 --- a/svc/pkg/team/ops/get/src/lib.rs +++ b/svc/pkg/team/ops/get/src/lib.rs @@ -61,7 +61,7 @@ async fn handle(ctx: OperationContext) -> GlobalResult::into); // Fetch all information relating to the profile image - let (profile_upload_complete_ts, profile_file_name) = { + let (profile_upload_complete_ts, profile_file_name, profile_provider) = { let upload = upload_res .uploads .iter() @@ -77,7 +77,7 @@ async fn handle(ctx: OperationContext) -> GlobalResult) -> GlobalResult) -> Gl let user_id = internal_unwrap!(ctx.user_id).as_uuid(); let user_set_status = - sqlx::query_as::<_, (Option,)>("SELECT user_set_status FROM user_presences") + sqlx::query_as::<_, (Option,)>("SELECT user_set_status FROM user_presences WHERE user_id = $1") + .bind(user_id) .fetch_optional(&crdb) .await? .and_then(|x| x.0) diff --git a/svc/pkg/user/db/user/migrations/20230515194135_search.up.sql b/svc/pkg/user/db/user/migrations/20230515194135_search.up.sql index 2464a763f8..d7e05c69b5 100644 --- a/svc/pkg/user/db/user/migrations/20230515194135_search.up.sql +++ b/svc/pkg/user/db/user/migrations/20230515194135_search.up.sql @@ -1,3 +1,3 @@ ALTER TABLE users - ADD COLUMN is_searchable BOOLEAN NOT NULL DEFAULT TRUE, + ADD COLUMN is_searchable BOOLEAN NOT NULL DEFAULT FALSE, DROP COLUMN display_name_len; diff --git a/svc/pkg/user/ops/avatar-upload-complete/Cargo.toml b/svc/pkg/user/ops/avatar-upload-complete/Cargo.toml index 6273ec563d..89d7ec0201 100644 --- a/svc/pkg/user/ops/avatar-upload-complete/Cargo.toml +++ b/svc/pkg/user/ops/avatar-upload-complete/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" upload-complete = { path = "../../../upload/ops/complete" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/get/Cargo.toml b/svc/pkg/user/ops/get/Cargo.toml index 263fe0a09e..b5c9e0fad0 100644 --- a/svc/pkg/user/ops/get/Cargo.toml +++ b/svc/pkg/user/ops/get/Cargo.toml @@ -14,7 +14,7 @@ upload-file-list = { path = "../../../upload/ops/file-list" } upload-get = { path = "../../../upload/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/get/src/lib.rs b/svc/pkg/user/ops/get/src/lib.rs index a7b63c829b..6f2cfef96a 100644 --- a/svc/pkg/user/ops/get/src/lib.rs +++ b/svc/pkg/user/ops/get/src/lib.rs @@ -51,15 +51,14 @@ pub async fn handle( .filter_map(|user| user.profile_id.map(Into::into)) .collect::>(); - let upload_res = op!([ctx] upload_get { - upload_ids: upload_ids.clone(), - }) - .await?; - - let files_res = op!([ctx] upload_file_list { - upload_ids: upload_ids.clone(), - }) - .await?; + let (upload_res, files_res) = tokio::try_join!( + op!([ctx] upload_get { + upload_ids: upload_ids.clone(), + }), + op!([ctx] upload_file_list { + upload_ids: upload_ids.clone(), + }) + )?; Ok(user::get::Response { users: users @@ -68,7 +67,7 @@ pub async fn handle( let profile_id = user.profile_id.map(Into::::into); // Fetch all information relating to the profile image - let (profile_upload_complete_ts, profile_file_name) = { + let (profile_upload_complete_ts, profile_file_name, profile_provider) = { let upload = upload_res .uploads .iter() @@ -84,7 +83,7 @@ pub async fn handle( .rsplit_once('/') .map(|(_, file_name)| file_name.to_owned()) .or(Some(file.path.clone())); - (upload.complete_ts, profile_file_name) + (upload.complete_ts, profile_file_name, Some(upload.provider)) } else { Default::default() } @@ -101,6 +100,7 @@ pub async fn handle( None }, profile_file_name, + profile_provider, join_ts: user.join_ts, bio: user.bio, is_admin: user.is_admin, diff --git a/svc/pkg/user/ops/mutual-friend-list/Cargo.toml b/svc/pkg/user/ops/mutual-friend-list/Cargo.toml index 07ace882f0..e3db3ec8b3 100644 --- a/svc/pkg/user/ops/mutual-friend-list/Cargo.toml +++ b/svc/pkg/user/ops/mutual-friend-list/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/pending-delete-toggle/Cargo.toml b/svc/pkg/user/ops/pending-delete-toggle/Cargo.toml index 1b3c73b317..451299d5bb 100644 --- a/svc/pkg/user/ops/pending-delete-toggle/Cargo.toml +++ b/svc/pkg/user/ops/pending-delete-toggle/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" user-identity-get = { path = "../../../user-identity/ops/get" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/profile-validate/Cargo.toml b/svc/pkg/user/ops/profile-validate/Cargo.toml index 7ae1fa55cd..9420a0015c 100644 --- a/svc/pkg/user/ops/profile-validate/Cargo.toml +++ b/svc/pkg/user/ops/profile-validate/Cargo.toml @@ -14,7 +14,7 @@ user-get = { path = "../get" } profanity-check = { path = "../../../profanity/ops/check" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/resolve-email/Cargo.toml b/svc/pkg/user/ops/resolve-email/Cargo.toml index ebef26efc1..f53b73a2f9 100644 --- a/svc/pkg/user/ops/resolve-email/Cargo.toml +++ b/svc/pkg/user/ops/resolve-email/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/search/Cargo.toml b/svc/pkg/user/ops/search/Cargo.toml index 7cdbd738a9..4b8f193d6a 100644 --- a/svc/pkg/user/ops/search/Cargo.toml +++ b/svc/pkg/user/ops/search/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" regex = "1.4" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/team-list/Cargo.toml b/svc/pkg/user/ops/team-list/Cargo.toml index c26c74e4e0..23eb6cf9b6 100644 --- a/svc/pkg/user/ops/team-list/Cargo.toml +++ b/svc/pkg/user/ops/team-list/Cargo.toml @@ -11,7 +11,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } prost = "0.10" [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/thread-typing-status-set/Cargo.toml b/svc/pkg/user/ops/thread-typing-status-set/Cargo.toml index 213e1555da..e852a2870c 100644 --- a/svc/pkg/user/ops/thread-typing-status-set/Cargo.toml +++ b/svc/pkg/user/ops/thread-typing-status-set/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" util-chat = { package = "rivet-util-chat", path = "../../../chat/util" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/ops/token-create/Cargo.toml b/svc/pkg/user/ops/token-create/Cargo.toml index fb99538b82..9489b5c4d2 100644 --- a/svc/pkg/user/ops/token-create/Cargo.toml +++ b/svc/pkg/user/ops/token-create/Cargo.toml @@ -13,7 +13,7 @@ prost = "0.10" token-create = { path = "../../../token/ops/create" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false [dev-dependencies] diff --git a/svc/pkg/user/standalone/delete-pending/Cargo.toml b/svc/pkg/user/standalone/delete-pending/Cargo.toml index 23bd8a7c8f..57486342fa 100644 --- a/svc/pkg/user/standalone/delete-pending/Cargo.toml +++ b/svc/pkg/user/standalone/delete-pending/Cargo.toml @@ -15,7 +15,7 @@ prost = "0.10" rivet-connection = { path = "../../../../../lib/connection" } rivet-pools = { path = "../../../../../lib/pools" } rivet-runtime = { path = "../../../../../lib/runtime" } -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } diff --git a/svc/pkg/user/standalone/search-user-gc/Cargo.toml b/svc/pkg/user/standalone/search-user-gc/Cargo.toml index 0a9253f34f..8ca7fd9dc8 100644 --- a/svc/pkg/user/standalone/search-user-gc/Cargo.toml +++ b/svc/pkg/user/standalone/search-user-gc/Cargo.toml @@ -14,7 +14,7 @@ indoc = "1.0" prost = "0.10" rivet-connection = { path = "../../../../../lib/connection" } rivet-runtime = { path = "../../../../../lib/runtime" } -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } diff --git a/svc/pkg/user/worker/adjectives.txt b/svc/pkg/user/worker/adjectives.txt index e86b5c3b9d..31d2b9b2dd 100644 --- a/svc/pkg/user/worker/adjectives.txt +++ b/svc/pkg/user/worker/adjectives.txt @@ -335,7 +335,6 @@ Plastic Plucky Poised Polite -Poor Premium Present Prickly diff --git a/svc/templates/api/Cargo.toml b/svc/templates/api/Cargo.toml index 51af8a5e8e..849d857349 100644 --- a/svc/templates/api/Cargo.toml +++ b/svc/templates/api/Cargo.toml @@ -20,7 +20,7 @@ rivet-pools = { path = "../../../lib/pools" } s3-util = { path = "../../../lib/s3-util" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "1.21" } +tokio = { version = "1.29" } tracing = "0.1" tracing-futures = "0.2" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } diff --git a/svc/templates/operation/Cargo.toml b/svc/templates/operation/Cargo.toml index e618941319..19049267c2 100644 --- a/svc/templates/operation/Cargo.toml +++ b/svc/templates/operation/Cargo.toml @@ -10,5 +10,5 @@ chirp-client = { path = "../../../../../lib/chirp/client" } rivet-operation = { path = "../../../../../lib/operation/core" } [dependencies.sqlx] -version = "0.6" +version = "0.7" default-features = false diff --git a/svc/templates/standalone/Cargo.toml b/svc/templates/standalone/Cargo.toml index 366e87d193..a791a00db1 100644 --- a/svc/templates/standalone/Cargo.toml +++ b/svc/templates/standalone/Cargo.toml @@ -10,7 +10,7 @@ chirp-client = { path = "../../../../../lib/chirp/client" } rivet-operation = { path = "../../../../../lib/operation/core" } rivet-connection = { path = "../../../../../lib/connection" } rivet-runtime = { path = "../../../../../lib/runtime" } -tokio = { version = "1.21", features = ["full"] } +tokio = { version = "1.29", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] }