Skip to content

Commit

Permalink
Merge pull request #80 from rl404/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@rl404
rl404 committed Jul 21, 2023
2 parents 801d7a1 + 44f4e3e commit 39a8c70
Show file tree
Hide file tree
Showing 20 changed files with 272 additions and 139 deletions.
22 changes: 11 additions & 11 deletions .github/renovate.json
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["config:base"],
"packageRules": [
{
"groupName": "all non-major dependencies",
"groupSlug": "all-minor-patch",
"matchPackagePatterns": ["*"],
"matchUpdateTypes": ["minor", "patch"]
}
],
"ignorePaths": ["vendor"]
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["config:base"],
"packageRules": [
{
"groupName": "all non-major dependencies",
"groupSlug": "all-minor-patch",
"matchPackagePatterns": ["*"],
"matchUpdateTypes": ["minor", "patch"]
}
],
"ignorePaths": ["vendor"]
}
24 changes: 12 additions & 12 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@ jobs:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Go environment
uses: actions/setup-go@v4
with:
go-version: 1.18
- name: Build
run: |
cd cmd/shimakaze
CGO_ENABLED=0 go build -a -installsuffix cgo -mod vendor -o shimakaze
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Go environment
uses: actions/setup-go@v4
with:
go-version: "1.20"
- name: Build
run: |
cd cmd/shimakaze
CGO_ENABLED=0 go build -a -installsuffix cgo -o shimakaze
4 changes: 1 addition & 3 deletions .github/workflows/publish-deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: publish & deploy
on:
push:
tags:
- '*'
- "*"

jobs:
publish_image:
Expand Down Expand Up @@ -83,5 +83,3 @@ jobs:
kubectl set image deployment ${{ secrets.GKE_CONSUMER_DEPLOYMENT_NAME }} ${{ secrets.GKE_CONSUMER_CONTAINER_NAME }}=rl404/shimakaze:${{ needs.publish_image.outputs.img_tag }}
kubectl set image cronjob ${{ secrets.GKE_CRON_FILL_DEPLOYMENT_NAME }} ${{ secrets.GKE_CRON_FILL_CONTAINER_NAME }}=rl404/shimakaze:${{ needs.publish_image.outputs.img_tag }}
kubectl set image cronjob ${{ secrets.GKE_CRON_UPDATE_DEPLOYMENT_NAME }} ${{ secrets.GKE_CRON_UPDATE_CONTAINER_NAME }}=rl404/shimakaze:${{ needs.publish_image.outputs.img_tag }}
16 changes: 8 additions & 8 deletions .github/workflows/tag.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ jobs:
name: Tag
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create tag
uses: mathieudutour/github-tag-action@v6.1
with:
github_token: ${{ secrets.CR_PAT }}
- name: Check out code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create tag
uses: mathieudutour/github-tag-action@v6.1
with:
github_token: ${{ secrets.CR_PAT }}
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ GO_CMD := go
GO_FMT := $(GO_CMD) fmt
GO_INSTALL := $(GO_CMD) install
GO_CLEAN := $(GO_CMD) clean
GO_BUILD := $(GO_CMD) build -mod vendor
GO_BUILD := $(GO_CMD) build

# Coverage output.
COVER_OUT := cover.out
Expand Down
121 changes: 65 additions & 56 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,35 @@ Shimakaze is [Vtuber Wikia](https://virtualyoutuber.fandom.com/wiki/Virtual_YouT
## Features

- Save vtuber details
- Vtuber data
- Vtuber channels data and videos
- Youtube
- Twitch
- Bilibili
- Niconico
- Vtuber data
- Vtuber channels data and videos
- Youtube
- Twitch
- Bilibili
- Niconico
- Save agency's vtuber list
- Auto update vtuber & agency data (cron)
- Interchangeable cache
- no cache
- inmemory
- [Redis](https://redis.io/)
- [Memcache](https://memcached.org/)
- no cache
- inmemory
- [Redis](https://redis.io/)
- [Memcache](https://memcached.org/)
- Interchangeable pubsub
- [NSQ](https://nsq.io/)
- [RabbitMQ](https://www.rabbitmq.com/)
- [Redis](https://redis.io/)
- [Google PubSub](https://cloud.google.com/pubsub)
- [NSQ](https://nsq.io/)
- [RabbitMQ](https://www.rabbitmq.com/)
- [Redis](https://redis.io/)
- [Google PubSub](https://cloud.google.com/pubsub)
- [Swagger](https://github.com/swaggo/swag)
- [Docker](https://www.docker.com/)
- [Newrelic](https://newrelic.com/) monitoring
- HTTP
- Cron
- Database
- Cache
- Pubsub
- External API
- HTTP
- Cron
- Database
- Cache
- Pubsub
- External API

*More will be coming soon...*
_More will be coming soon..._

## Requirement

Expand All @@ -57,18 +57,22 @@ Shimakaze is [Vtuber Wikia](https://virtualyoutuber.fandom.com/wiki/Virtual_YouT
### Without [Docker](https://www.docker.com/) & [Docker Compose](https://docs.docker.com/compose/)

1. Clone the repository.

```sh
git clone github.com/rl404/shimakaze
```

2. Rename `.env.sample` to `.env` and modify the values according to your setup.
3. Run. You need at least 2 consoles/terminals.

```sh
# Run the API.
make

# Run the consumer.
make consumer
```

6. [localhost:45001](http://localhost:45001) is ready (port may varies depend on your `.env`).

#### Other commands
Expand All @@ -84,26 +88,32 @@ make cron-fill
### With [Docker](https://www.docker.com/) & [Docker Compose](https://docs.docker.com/compose/)

1. Clone the repository.

```sh
git clone github.com/rl404/shimakaze
```

2. Rename `.env.sample` to `.env` and modify the values according to your setup.
3. Get docker image.

```sh
# Pull existing image.
docker pull rl404/shimakaze

# Or build your own.
make docker-build
```

4. Run the container. You need at least 2 consoles/terminals.

```sh
# Run the API.
make docker-api

# Run the consumer.
make docker-consumer
```

5. [localhost:45001](http://localhost:45001) is ready (port may varies depend on your `.env`).

#### Other commands
Expand All @@ -121,43 +131,42 @@ make docker-stop

## Environment Variables

Env | Default | Description
--- | :---: | ---
`SHIMAKAZE_APP_ENV` | `dev` | Environment type (`dev`/`prod`).
`SHIMAKAZE_HTTP_PORT` | `45001` | HTTP server port.
`SHIMAKAZE_HTTP_READ_TIMEOUT` | `5s` | HTTP read timeout.
`SHIMAKAZE_HTTP_WRITE_TIMEOUT` | `5s` | HTTP write timeout.
`SHIMAKAZE_HTTP_GRACEFUL_TIMEOUT` | `10s` | HTTP graceful timeout.
`SHIMAKAZE_CACHE_DIALECT` | `inmemory` | Cache type (`nocache`/`redis`/`inmemory`/`memcache`)
`SHIMAKAZE_CACHE_ADDRESS` | | Cache address.
`SHIMAKAZE_CACHE_PASSWORD` | | Cache password.
`SHIMAKAZE_CACHE_TIME` | `24h` | Cache time.
`SHIMAKAZE_DB_ADDRESS` | `mongodb://localhost:27017` | Database address with port.
`SHIMAKAZE_DB_NAME` | `shimakaze` | Database name.
`SHIMAKAZE_DB_USER` | | Database username.
`SHIMAKAZE_DB_PASSWORD` | | Database password.
`SHIMAKAZE_PUBSUB_DIALECT` | `rabbitmq` | Pubsub type (`nsq`/`rabbitmq`/`redis`/`google`)
`SHIMAKAZE_PUBSUB_ADDRESS` | | Pubsub address (if you are using `google`, this will be your google project id).
`SHIMAKAZE_PUBSUB_PASSWORD` | | Pubsub password (if you are using `google`, this will be the content of your google service account json).
`SHIMAKAZE_CRON_UPDATE_LIMIT` | `10` | Vtuber & agency count limit when updating old data.
`SHIMAKAZE_CRON_FILL_LIMIT` | `10` | Vtuber & agency count limit when filling missing data.
`SHIMAKAZE_CRON_AGENCY_AGE` | `7` | Age of old agency data (in days).
`SHIMAKAZE_CRON_ACTIVE_AGE` | `1` | Age of old active vtuber data (in days).
`SHIMAKAZE_CRON_RETIRED_AGE` | `7` | Age of old retired vtuber data (in days).
`SHIMAKAZE_NEWRELIC_NAME` | `shimakaze` | Newrelic application name.
`SHIMAKAZE_NEWRELIC_LICENSE_KEY` | | Newrelic license key.
`SHIMAKAZE_YOUTUBE_KEY` | | Youtube API key.
`SHIMAKAZE_YOUTUBE_MAX_AGE` | `60` | Age limit of youtube videos (in days).
`SHIMAKAZE_TWITCH_CLIENT_ID` | | Twitch client id.
`SHIMAKAZE_TWITCH_CLIENT_SECRET` | | Twitch client secret.
`SHIMAKAZE_TWITCH_MAX_AGE` | `60` | Age limit of twitch videos (in days).
`SHIMAKAZE_BILIBILI_MAX_AGE` | `60` | Age limit of bilibili videos (in days).
`SHIMAKAZE_NICONICO_MAX_AGE` | `60` | Age limit of niconico videos (in days).

| Env | Default | Description |
| --------------------------------- | :-------------------------: | ---------------------------------------------------------------------------------------------------------- |
| `SHIMAKAZE_APP_ENV` | `dev` | Environment type (`dev`/`prod`). |
| `SHIMAKAZE_HTTP_PORT` | `45001` | HTTP server port. |
| `SHIMAKAZE_HTTP_READ_TIMEOUT` | `5s` | HTTP read timeout. |
| `SHIMAKAZE_HTTP_WRITE_TIMEOUT` | `5s` | HTTP write timeout. |
| `SHIMAKAZE_HTTP_GRACEFUL_TIMEOUT` | `10s` | HTTP graceful timeout. |
| `SHIMAKAZE_CACHE_DIALECT` | `inmemory` | Cache type (`nocache`/`redis`/`inmemory`/`memcache`) |
| `SHIMAKAZE_CACHE_ADDRESS` | | Cache address. |
| `SHIMAKAZE_CACHE_PASSWORD` | | Cache password. |
| `SHIMAKAZE_CACHE_TIME` | `24h` | Cache time. |
| `SHIMAKAZE_DB_ADDRESS` | `mongodb://localhost:27017` | Database address with port. |
| `SHIMAKAZE_DB_NAME` | `shimakaze` | Database name. |
| `SHIMAKAZE_DB_USER` | | Database username. |
| `SHIMAKAZE_DB_PASSWORD` | | Database password. |
| `SHIMAKAZE_PUBSUB_DIALECT` | `rabbitmq` | Pubsub type (`nsq`/`rabbitmq`/`redis`/`google`) |
| `SHIMAKAZE_PUBSUB_ADDRESS` | | Pubsub address (if you are using `google`, this will be your google project id). |
| `SHIMAKAZE_PUBSUB_PASSWORD` | | Pubsub password (if you are using `google`, this will be the content of your google service account json). |
| `SHIMAKAZE_CRON_UPDATE_LIMIT` | `10` | Vtuber & agency count limit when updating old data. |
| `SHIMAKAZE_CRON_FILL_LIMIT` | `10` | Vtuber & agency count limit when filling missing data. |
| `SHIMAKAZE_CRON_AGENCY_AGE` | `7` | Age of old agency data (in days). |
| `SHIMAKAZE_CRON_ACTIVE_AGE` | `1` | Age of old active vtuber data (in days). |
| `SHIMAKAZE_CRON_RETIRED_AGE` | `7` | Age of old retired vtuber data (in days). |
| `SHIMAKAZE_NEWRELIC_NAME` | `shimakaze` | Newrelic application name. |
| `SHIMAKAZE_NEWRELIC_LICENSE_KEY` | | Newrelic license key. |
| `SHIMAKAZE_YOUTUBE_KEY` | | Youtube API key. |
| `SHIMAKAZE_YOUTUBE_MAX_AGE` | `60` | Age limit of youtube videos (in days). |
| `SHIMAKAZE_TWITCH_CLIENT_ID` | | Twitch client id. |
| `SHIMAKAZE_TWITCH_CLIENT_SECRET` | | Twitch client secret. |
| `SHIMAKAZE_TWITCH_MAX_AGE` | `60` | Age limit of twitch videos (in days). |
| `SHIMAKAZE_BILIBILI_MAX_AGE` | `60` | Age limit of bilibili videos (in days). |
| `SHIMAKAZE_NICONICO_MAX_AGE` | `60` | Age limit of niconico videos (in days). |

## Trivia

[Shimakaze](https://en.wikipedia.org/wiki/Japanese_destroyer_Shimakaze_(1942))'s name is taken from one of the fastest japanese destroyer. Also, [exists](https://en.kancollewiki.net/Shimakaze) in Kantai Collection games and manga.
[Shimakaze](<https://en.wikipedia.org/wiki/Japanese_destroyer_Shimakaze_(1942)>)'s name is taken from one of the fastest japanese destroyer. Also, [exists](https://en.kancollewiki.net/Shimakaze) in Kantai Collection games and manga.

## Disclaimer

Expand All @@ -169,4 +178,4 @@ All data belong to their respective copyrights owners, shimakaze does not have a

MIT License

Copyright (c) 2023 Axel
Copyright (c) 2023 Axel
2 changes: 1 addition & 1 deletion build/package/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ LABEL stage="shimakaze_builder"
WORKDIR /shimakaze
COPY . .
WORKDIR /shimakaze/cmd/shimakaze
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -mod vendor -o shimakaze
RUN CGO_ENABLED=0 go build -a -installsuffix cgo -o shimakaze

# New stage from scratch
FROM alpine:3.18
Expand Down
18 changes: 9 additions & 9 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/rl404/shimakaze

go 1.18
go 1.20

require (
github.com/PuerkitoBio/goquery v1.8.1
Expand All @@ -10,7 +10,7 @@ require (
github.com/newrelic/go-agent/v3 v3.23.1
github.com/newrelic/go-agent/v3/integrations/nrmongo v1.1.0
github.com/nicklaw5/helix/v2 v2.23.1
github.com/rl404/fairy v0.22.3
github.com/rl404/fairy v0.23.0
github.com/spf13/cobra v1.7.0
github.com/swaggo/http-swagger/v2 v2.0.1
github.com/swaggo/swag v1.16.1
Expand All @@ -21,7 +21,7 @@ require (

require (
cloud.google.com/go v0.110.2 // indirect
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
cloud.google.com/go/pubsub v1.32.0 // indirect
Expand Down Expand Up @@ -49,7 +49,7 @@ require (
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gosimple/slug v1.13.1 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
Expand Down Expand Up @@ -82,15 +82,15 @@ require (
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.9.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/tools v0.7.0 // indirect
google.golang.org/api v0.130.0 // indirect
google.golang.org/api v0.131.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
google.golang.org/grpc v1.56.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading

0 comments on commit 39a8c70

Please sign in to comment.