From 3584494cf62e5e3067c5984bfdbaeec5659aec8e Mon Sep 17 00:00:00 2001 From: Emil Shakirov Date: Mon, 15 Jun 2020 10:16:40 +0200 Subject: [PATCH 1/4] Add Timecode#UserID --- cmd/db_migrations.go | 6 ++++- cmd/timecode_repository.go | 10 ++++++++- cmd/timecodes_controller.go | 6 +++++ cmd/timecodes_controller_test.go | 38 ++++++++++---------------------- cmd/user_repository.go | 8 +++++++ 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/cmd/db_migrations.go b/cmd/db_migrations.go index ce7fbec..18661cc 100644 --- a/cmd/db_migrations.go +++ b/cmd/db_migrations.go @@ -1,6 +1,8 @@ package main -import "github.com/jinzhu/gorm" +import ( + "github.com/jinzhu/gorm" +) func runMigrations(db *gorm.DB) { applyTimecodesMigrations(db) @@ -22,6 +24,8 @@ func applyTimecodesMigrations(db *gorm.DB) { func applyUsersMigrations(db *gorm.DB) { db.AutoMigrate(&User{}) + + getAdminUser(db) } func applyTimecodeLikesMigrations(db *gorm.DB) { diff --git a/cmd/timecode_repository.go b/cmd/timecode_repository.go index 4aa54f3..48b4449 100644 --- a/cmd/timecode_repository.go +++ b/cmd/timecode_repository.go @@ -14,6 +14,7 @@ type Timecode struct { Seconds int `json:"seconds" gorm:"not null"` VideoID string `json:"videoId" gorm:"not null;index"` Likes []TimecodeLike `json:"likes" gorm:"foreignkey:TimecodeID"` + UserID uint `json:"userId"` } type TimecodeRepository interface { @@ -57,7 +58,14 @@ func (repo *DBTimecodeRepository) CreateFromParsedCodes(parsedTimecodes []timeco seen[key] = struct{}{} - timecode := &Timecode{Seconds: code.Seconds, VideoID: videoId, Description: code.Description} + user := getAdminUser(repo.DB) + + timecode := &Timecode{ + Seconds: code.Seconds, + VideoID: videoId, + Description: code.Description, + UserID: user.ID, + } err := repo.DB.Create(timecode).Error if err != nil { diff --git a/cmd/timecodes_controller.go b/cmd/timecodes_controller.go index 6142b69..9be1911 100644 --- a/cmd/timecodes_controller.go +++ b/cmd/timecodes_controller.go @@ -24,6 +24,7 @@ type TimecodeJSON struct { LikedByMe bool `json:"likedByMe,omitempty"` Seconds int `json:"seconds"` VideoID string `json:"videoId"` + UserID uint `json:"userId,omitempty"` } // GET /timecodes/{videoId} @@ -63,6 +64,7 @@ func handleCreateTimecode(c *Container, w http.ResponseWriter, r *http.Request) Description: timecodeRequest.Description, Seconds: timecodeParser.ParseSeconds(timecodeRequest.RawSeconds), VideoID: timecodeRequest.VideoID, + UserID: currentUser.ID, } _, err = c.TimecodeRepository.Create(timecode) @@ -77,8 +79,11 @@ func handleCreateTimecode(c *Container, w http.ResponseWriter, r *http.Request) func serializeTimecode(timecode *Timecode, currentUser *User) (timecodeJSON *TimecodeJSON) { var likedByMe bool + var userId uint + if currentUser != nil { likedByMe = getLikedByMe(timecode.Likes, currentUser.ID) + userId = timecode.UserID } return &TimecodeJSON{ @@ -88,6 +93,7 @@ func serializeTimecode(timecode *Timecode, currentUser *User) (timecodeJSON *Tim LikedByMe: likedByMe, Seconds: timecode.Seconds, VideoID: timecode.VideoID, + UserID: userId, } } diff --git a/cmd/timecodes_controller_test.go b/cmd/timecodes_controller_test.go index a872f24..fe6c14e 100644 --- a/cmd/timecodes_controller_test.go +++ b/cmd/timecodes_controller_test.go @@ -27,28 +27,13 @@ type MockTimecodeRepository struct { func (m *MockTimecodeRepository) FindByVideoId(videoID string) *[]*Timecode { args := m.Called(videoID) - collection := args.Get(0).(*[]*Timecode) - if videoID == "no-items" { - return collection - } - - collection = &[]*Timecode{{}, {}, {Likes: []TimecodeLike{{UserID: 1}}}} - - return collection + return args.Get(0).(*[]*Timecode) } func (m *MockTimecodeRepository) Create(timecode *Timecode) (*Timecode, error) { args := m.Called(timecode) - err := args.Error(1) - - if len(timecode.Description) == 0 { - return nil, err - } - - _ = args.Get(0).(*Timecode) - - return timecode, nil + return args.Get(0).(*Timecode), args.Error(1) } func (m *MockTimecodeRepository) CreateFromParsedCodes(parsedCodes []timecodeParser.ParsedTimeCode, videoId string) *[]*Timecode { @@ -64,17 +49,13 @@ type mockYT struct { func (m *mockYT) FetchVideoDescription(videoId string) string { args := m.Called(videoId) - _ = args.Get(0).(string) - - return "description" + return args.Get(0).(string) } func (m *mockYT) FetchVideoComments(videoId string) []string { args := m.Called(videoId) - _ = args.Get(0).([]string) - - return []string{"comment one", "comment two"} + return args.Get(0).([]string) } func Test_handleGetTimecodes(t *testing.T) { @@ -122,7 +103,12 @@ func Test_handleCreateTimecode(t *testing.T) { currentUser.ID = 1 t.Run("when request params are valid", func(t *testing.T) { - timecode := &Timecode{VideoID: "video-id", Seconds: 71, Description: "ABC"} + timecode := &Timecode{ + VideoID: "video-id", + Seconds: 71, + Description: "ABC", + UserID: currentUser.ID, + } mockTimecodeRepo.On("Create", timecode).Return(timecode, nil) @@ -136,9 +122,9 @@ func Test_handleCreateTimecode(t *testing.T) { }) t.Run("when request params are invalid", func(t *testing.T) { - timecode := &Timecode{VideoID: "video-id", Seconds: 71, Description: ""} + timecode := &Timecode{VideoID: "video-id", Seconds: 71, Description: "", UserID: currentUser.ID} - mockTimecodeRepo.On("Create", timecode).Return(nil, errors.New("")) + mockTimecodeRepo.On("Create", timecode).Return(&Timecode{}, errors.New("")) params := []byte(`{ "videoId": "video-id", "seconds": "1:11", "description": "" }`) req, _ := http.NewRequest(http.MethodPost, "/auth/timecodes", bytes.NewBuffer(params)) diff --git a/cmd/user_repository.go b/cmd/user_repository.go index 8bfecfd..14fbdfd 100644 --- a/cmd/user_repository.go +++ b/cmd/user_repository.go @@ -1,6 +1,7 @@ package main import ( + "os" googleAPI "timecodes/cmd/google_api" "github.com/jinzhu/gorm" @@ -32,3 +33,10 @@ func (repo *DBUserRepository) FindOrCreateByGoogleInfo(userInfo *googleAPI.UserI return user } + +func getAdminUser(db *gorm.DB) *User { + adminUser := &User{Email: os.Getenv("ADMIN_EMAIL"), GoogleID: os.Getenv("ADMIN_GOOGLE_ID")} + db.FirstOrCreate(adminUser) + + return adminUser +} From 9fbac6345a9c876ad4185b362ee0addab85fc36d Mon Sep 17 00:00:00 2001 From: Emil Shakirov Date: Sat, 25 Jul 2020 14:10:08 +0200 Subject: [PATCH 2/4] Refactor deployment configuration --- .env.example | 5 +++ .env.test | 2 ++ .github/workflows/ci.yml | 31 ++++++++++++++-- .github/workflows/ssh_deploy.yml | 25 ------------- docker-compose.yml | 61 +++++++++++++++++++------------- support/nginx/app.conf | 30 ---------------- support/test.sh | 3 ++ 7 files changed, 75 insertions(+), 82 deletions(-) delete mode 100644 .github/workflows/ssh_deploy.yml delete mode 100644 support/nginx/app.conf create mode 100755 support/test.sh diff --git a/.env.example b/.env.example index ee205ae..bd9463a 100644 --- a/.env.example +++ b/.env.example @@ -8,3 +8,8 @@ PG_PORT=5432 POSTGRES_DB=timecodes_development GOOGLE_API_KEY= + +POSTGRES_USER= +POSTGRES_PASSWORD= + +APP_HOST= diff --git a/.env.test b/.env.test index bbdcd55..8dc0eaa 100644 --- a/.env.test +++ b/.env.test @@ -5,6 +5,8 @@ PG_PASSWORD=postgres PG_HOST=db PG_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres POSTGRES_DB=timecodes_test GOOGLE_API_KEY= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2166f7..01faeda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,12 +30,15 @@ jobs: push_image_and_stages: false pull_image_and_stages: false - name: Build docker image for tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cp .env.example .env - echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u vaihtovirta --password-stdin + cp .env.test .env + docker login docker.pkg.github.com -u vaihtovirta -p $GITHUB_TOKEN + docker network create traefik-public docker-compose build --pull app_test - name: Run tests - run: docker-compose run --rm app_test go test ./... -covermode=count -coverprofile=tmp/coverage.out + run: docker-compose run --rm app_test sh -c "./support/test.sh" - name: Convert coverage to lcov uses: jandelgado/gcov2lcov-action@v1.0.2 with: @@ -46,3 +49,25 @@ jobs: with: github-token: ${{ secrets.github_token }} path-to-lcov: coverage.lcov + deployment: + if: github.ref == 'refs/heads/master' + needs: [lint, test] + runs-on: ubuntu-latest + steps: + - name: Connect by ssh and rebuild docker image + uses: appleboy/ssh-action@master + env: + DEPLOY_PASSWORD: ${{ secrets.PASSWORD }} + with: + host: ${{ secrets.HOST }} + username: ${{ secrets.USERNAME }} + passphrase: ${{ secrets.PASSPHRASE }} + key: ${{ secrets.PRIVATE_KEY }} + envs: DEPLOY_PASSWORD + script: | + cd /home/deploy/applications/timecodes/timecodes-api + git fetch --all + git reset --hard origin/master + export $(egrep -v '^#' .env.production | xargs) + docker-compose -f docker-compose.prod.yml pull app + docker-compose -f docker-compose.prod.yml up -d diff --git a/.github/workflows/ssh_deploy.yml b/.github/workflows/ssh_deploy.yml deleted file mode 100644 index 7775c1f..0000000 --- a/.github/workflows/ssh_deploy.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Deploy to Digital Ocean -on: - push: - branches: - - master -jobs: - deployment: - runs-on: ubuntu-latest - steps: - - name: Connect by ssh and rebuild docker image - uses: appleboy/ssh-action@master - env: - DEPLOY_PASSWORD: ${{ secrets.PASSWORD }} - with: - host: ${{ secrets.HOST }} - username: ${{ secrets.USERNAME }} - passphrase: ${{ secrets.PASSPHRASE }} - key: ${{ secrets.PRIVATE_KEY }} - envs: DEPLOY_PASSWORD - script: | - cd /home/deploy/applications/timecodes/timecodes-api - git checkout master - git pull - echo $DEPLOY_PASSWORD | sudo -S docker-compose build app - echo $DEPLOY_PASSWORD | sudo -S docker-compose up -d app diff --git a/docker-compose.yml b/docker-compose.yml index 9a676c7..7b3b249 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,14 @@ version: "3.5" services: + # Base app_base: image: docker.pkg.github.com/letscode-io/timecodes-api/timecodes-api_base:latest build: context: . dockerfile: Dockerfile.base + + # Test app_test: build: context: . @@ -17,8 +20,13 @@ services: - CGO_ENABLED=0 env_file: - .env.test + networks: + - default volumes: - ./tmp:/usr/src/app/tmp + - .:/usr/src/app + + # Dev app_dev: build: context: . @@ -31,6 +39,8 @@ services: - "8080:8080" volumes: - .:/usr/src/app + + # Production app: restart: always build: @@ -39,36 +49,39 @@ services: target: final depends_on: - db - - nginx - - certbot env_file: - .env ports: - - "8080:8080" + - "8080" + labels: + - "traefik.enable=true" + - "traefik.docker.network=traefik-public" + + - traefik.http.routers.timecodes-http.rule=Host(`${APP_HOST}`) + - "traefik.http.services.timecodes-service.loadbalancer.server.port=8080" + + - traefik.http.routers.timecodes-http.entrypoints=http + - traefik.http.routers.timecodes-http.service=timecodes-service + - traefik.http.routers.timecodes-http.middlewares=https-redirect + + - traefik.http.routers.timecodes-https.rule=Host(`${APP_HOST}`) + - traefik.http.routers.timecodes-https.entrypoints=https + - traefik.http.routers.timecodes-https.tls=true + - traefik.http.routers.timecodes-https.tls.certresolver=le + networks: + - traefik-public + - default + + # Database db: image: postgres:12-alpine - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres env_file: - .env ports: - 9000:5432 - nginx: - image: nginx:latest - restart: on-failure - ports: - - "80:80" - - "443:443" - volumes: - - ./support/nginx:/etc/nginx/conf.d - - ./support/certbot/conf:/etc/letsencrypt - - ./support/certbot/www:/var/www/certbot - command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'" - certbot: - image: certbot/certbot - restart: unless-stopped - volumes: - - ./support/certbot/conf:/etc/letsencrypt - - ./support/certbot/www:/var/www/certbot - entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" + networks: + - default + +networks: + traefik-public: + external: true diff --git a/support/nginx/app.conf b/support/nginx/app.conf deleted file mode 100644 index 3e9251f..0000000 --- a/support/nginx/app.conf +++ /dev/null @@ -1,30 +0,0 @@ -server { - server_name timecodes.letscode.io; - listen 80; - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - - location / { - return 301 https://$host$request_uri; - } -} - -server { - server_name timecodes.letscode.io; - server_tokens off; - listen 443 ssl; - - ssl_certificate /etc/letsencrypt/live/timecodes.letscode.io/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/timecodes.letscode.io/privkey.pem; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass "http://app:8080"; - } -} diff --git a/support/test.sh b/support/test.sh new file mode 100755 index 0000000..e195c0f --- /dev/null +++ b/support/test.sh @@ -0,0 +1,3 @@ +while ! nc -z db 5432; do sleep 1; done; + +go test ./... -covermode=count -coverprofile=tmp/coverage.out From 94fc97049ee20a17b545a39def8750d6cb16ba83 Mon Sep 17 00:00:00 2001 From: Emil Shakirov Date: Sat, 25 Jul 2020 14:10:26 +0200 Subject: [PATCH 3/4] Fix mocks usage in likes controller tests --- cmd/timecode_likes_controller_test.go | 45 +++++++-------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/cmd/timecode_likes_controller_test.go b/cmd/timecode_likes_controller_test.go index 5a4dfce..9cc4105 100644 --- a/cmd/timecode_likes_controller_test.go +++ b/cmd/timecode_likes_controller_test.go @@ -27,43 +27,26 @@ type MockTimecodeLikeRepository struct { func (mr *MockTimecodeLikeRepository) Create(like *TimecodeLike, userID uint) (*TimecodeLike, error) { args := mr.Called(like, userID) - err := args.Error(1) - like.UserID = userID - - if like.TimecodeID == invalidTimecodeID { - return nil, err - } - - _ = args.Get(0).(*TimecodeLike) - - return like, nil + return args.Get(0).(*TimecodeLike), args.Error(1) } func (mr *MockTimecodeLikeRepository) Delete(like *TimecodeLike, userID uint) (*TimecodeLike, error) { args := mr.Called(like, userID) - err := args.Error(1) - - like.UserID = userID - if like.TimecodeID == invalidTimecodeID { - return nil, err - } - - _ = args.Get(0).(*TimecodeLike) - - return like, nil + return args.Get(0).(*TimecodeLike), args.Error(1) } func Test_handleCreateTimecodeLike(t *testing.T) { url := "/auth/timecode_likes" + currentUser := &User{} + currentUser.ID = 1 t.Run("when creation is successful", func(t *testing.T) { - currentUser := &User{} - currentUser.ID = 1 + likeParams := &TimecodeLike{TimecodeID: 5} + like := &TimecodeLike{TimecodeID: 5, UserID: currentUser.ID} - like := &TimecodeLike{TimecodeID: 5} - mockTKRepo.On("Create", like, currentUser.ID).Return(like, nil) + mockTKRepo.On("Create", likeParams, currentUser.ID).Return(like, nil) params := []byte(`{ "timecodeId": 5 }`) req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(params)) @@ -75,11 +58,8 @@ func Test_handleCreateTimecodeLike(t *testing.T) { }) t.Run("when creation has been failed", func(t *testing.T) { - currentUser := &User{} - currentUser.ID = 1 - like := &TimecodeLike{TimecodeID: invalidTimecodeID} - mockTKRepo.On("Create", like, currentUser.ID).Return(nil, errors.New("")) + mockTKRepo.On("Create", like, currentUser.ID).Return(&TimecodeLike{}, errors.New("")) params := []byte(fmt.Sprintf(`{ "timecodeId": %d }`, invalidTimecodeID)) req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(params)) @@ -103,10 +83,10 @@ func Test_handleCreateTimecodeLike(t *testing.T) { func Test_handleDeleteTimecodeLike(t *testing.T) { url := "/auth/timecode_likes" + currentUser := &User{} + currentUser.ID = 1 t.Run("when deletion is successful", func(t *testing.T) { - currentUser := &User{} - currentUser.ID = 1 like := &TimecodeLike{TimecodeID: 5} mockTKRepo.On("Delete", like, currentUser.ID).Return(like, nil) @@ -121,11 +101,8 @@ func Test_handleDeleteTimecodeLike(t *testing.T) { }) t.Run("when deletion has been failed", func(t *testing.T) { - currentUser := &User{} - currentUser.ID = 1 - like := &TimecodeLike{TimecodeID: invalidTimecodeID} - mockTKRepo.On("Delete", like, currentUser.ID).Return(nil, errors.New("")) + mockTKRepo.On("Delete", like, currentUser.ID).Return(&TimecodeLike{}, errors.New("")) params := []byte(fmt.Sprintf(`{ "timecodeId": %d }`, invalidTimecodeID)) req, _ := http.NewRequest(http.MethodDelete, url, bytes.NewBuffer(params)) From e2a37aa4d5c8cceb11ed7e2993b39caa7ce5ad65 Mon Sep 17 00:00:00 2001 From: Emil Shakirov Date: Sat, 25 Jul 2020 14:15:36 +0200 Subject: [PATCH 4/4] Clean up CI pipeline --- .github/workflows/ci.yml | 7 ++----- support/test.sh | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100755 support/test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01faeda..193bbf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: docker network create traefik-public docker-compose build --pull app_test - name: Run tests - run: docker-compose run --rm app_test sh -c "./support/test.sh" + run: docker-compose run --rm app_test go test ./... -covermode=count -coverprofile=tmp/coverage.out - name: Convert coverage to lcov uses: jandelgado/gcov2lcov-action@v1.0.2 with: @@ -56,18 +56,15 @@ jobs: steps: - name: Connect by ssh and rebuild docker image uses: appleboy/ssh-action@master - env: - DEPLOY_PASSWORD: ${{ secrets.PASSWORD }} with: host: ${{ secrets.HOST }} username: ${{ secrets.USERNAME }} passphrase: ${{ secrets.PASSPHRASE }} key: ${{ secrets.PRIVATE_KEY }} - envs: DEPLOY_PASSWORD script: | cd /home/deploy/applications/timecodes/timecodes-api git fetch --all git reset --hard origin/master - export $(egrep -v '^#' .env.production | xargs) + dotenvp docker-compose -f docker-compose.prod.yml pull app docker-compose -f docker-compose.prod.yml up -d diff --git a/support/test.sh b/support/test.sh deleted file mode 100755 index e195c0f..0000000 --- a/support/test.sh +++ /dev/null @@ -1,3 +0,0 @@ -while ! nc -z db 5432; do sleep 1; done; - -go test ./... -covermode=count -coverprofile=tmp/coverage.out