Add gitlab ci support

.gitlab-ci.yml

@@ -0,0 +1,100 @@
+image: chocobozzz/peertube-ci:10
+
+stages:
+  - build-and-lint
+  - test
+  - nightly
+
+before_script:
+  - 'sed -i -z "s/database:\n  hostname: ''localhost''/database:\n  hostname: ''postgres''/" config/test.yaml'
+  - 'sed -i -z "s/redis:\n  hostname: ''localhost''/redis:\n  hostname: ''redis''/" config/test.yaml'
+  - if [[ $CI_JOB_STAGE == "test" ]]; then psql -c "create user peertube with password 'peertube';"; fi
+  - NOCLIENT=1 yarn install --pure-lockfile --cache-folder .yarn-cache
+
+cache:
+  key: yarn
+  paths:
+    - .yarn-cache
+    - cached-fixtures
+
+###
+## Jobs templates
+#
+#.build-and-lint: &build-and-lint
+#  stage: build-and-lint
+#
+#.tests: &tests
+#  stage: test
+#  dependencies:
+#    - build-server
+#  services:
+#    - name: postgres:9.6
+#      alias: postgres
+#    - name: redis:latest
+#      alias: redis
+#  variables:
+#    PGHOST: postgres
+#    PGUSER: postgres
+#    REDIS_HOST: redis
+#  artifacts:
+#    expire_in: 1 day
+#    paths:
+#      - test*/logs
+#    when: always
+#
+####
+### Build and lint
+##
+#build-server:
+#  <<: *build-and-lint
+#  artifacts:
+#    expire_in: 5h
+#    paths:
+#      - dist/
+#  script:
+#    - npm run build:server
+#
+#lint:
+#  <<: *build-and-lint
+#  script:
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
+#    - npm run ci -- "lint"
+#
+####
+### Tests
+#
+#test-misc:
+#  <<: *tests
+#  script:
+#    - yarn install --pure-lockfile --cache-folder .yarn-cache
+#    - npm run ci -- "misc"
+#
+#test-cli:
+#  <<: *tests
+#  retry:
+#    max: 1
+#  script:
+#    - npm run ci -- "cli"
+#
+#api:
+#  <<: *tests
+#  parallel: 4
+#  retry:
+#    max: 1
+#  script:
+#    - NODE_PENDING_JOB_WAIT=1000 npm run ci -- api-$CI_NODE_INDEX
+
+build-nightly:
+  stage: nightly
+  only:
+    - schedules
+  script:
+    - yarn install --pure-lockfile --cache-folder .yarn-cache
+    - npm run nightly
+    - mkdir "${HOME}/.ssh"
+    - chmod 700 "${HOME}/.ssh"
+    - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi
+    - eval `ssh-agent -s`
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi
+    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi
+

.travis.yml

@@ -47,7 +47,7 @@ matrix:
   - env: TEST_SUITE=lint
 
 script:
-  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run travis -- "$TEST_SUITE"
+  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE"
 
 after_failure:
   - cat test1/logs/peertube.log

README.md

@@ -45,8 +45,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
 
   <br />
 
-  <a href="https://travis-ci.org/Chocobozzz/PeerTube">
-    <img src="https://travis-ci.org/Chocobozzz/PeerTube.svg?branch=develop" alt="Build Status" />
+  <a href="https://framagit.org/framasoft/peertube/PeerTube/commits/develop">
+    <img alt="pipeline status" src="https://framagit.org/framasoft/peertube/PeerTube/badges/develop/pipeline.svg" />
   </a>
 
   <a href="https://david-dm.org/Chocobozzz/PeerTube">

config/test.yaml

@@ -20,6 +20,9 @@ database:
   hostname: 'localhost'
   port: 5432
 
+redis:
+  hostname: 'localhost'
+
 smtp:
   hostname: null
   port: 1025

package.json

@@ -60,17 +60,19 @@
     "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)",
     "tsc": "tsc",
     "commander": "commander",
-    "lint": "npm run travis -- lint",
+    "lint": "npm run ci -- lint",
     "ng": "ng",
     "nodemon": "nodemon",
     "ts-node": "ts-node",
     "tslint": "tslint",
     "concurrently": "concurrently",
+    "mocha-parallel-tests": "mocha-parallel-tests",
     "sasslint": "sass-lint --verbose --no-exit",
     "sasslint:fix": "sass-lint-auto-fix -c .sass-lint.yml --verbose",
     "mocha": "mocha",
-    "travis": "scripty",
+    "ci": "scripty",
     "release": "scripty",
+    "nightly": "scripty",
     "client-report": "scripty"
   },
   "husky": {
@@ -93,7 +95,7 @@
     ]
   },
   "resolutions": {
-    "@types/bluebird": "3.5.21"
+    "@types/bluebird": "3.5.27"
   },
   "dependencies": {
     "apicache": "^1.4.0",
@@ -206,7 +208,7 @@
     "maildev": "^1.0.0-rc3",
     "marked-man": "^0.6.0",
     "mocha": "^6.0.0",
-    "mocha-parallel-tests": "^2.1.0",
+    "mocha-parallel-tests": "^2.2.1",
     "nodemon": "^1.18.6",
     "sass-lint": "^1.12.1",
     "source-map-support": "^0.5.0",

scripts/travis.sh → scripts/ci.sh

@@ -22,16 +22,16 @@ elif [ "$1" = "cli" ]; then
     mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
 elif [ "$1" = "api-1" ]; then
     npm run build:server
-    sh ./server/tests/api/travis-1.sh 2
+    sh ./server/tests/api/ci-1.sh 2
 elif [ "$1" = "api-2" ]; then
     npm run build:server
-    sh ./server/tests/api/travis-2.sh 2
+    sh ./server/tests/api/ci-2.sh 2
 elif [ "$1" = "api-3" ]; then
     npm run build:server
-    sh ./server/tests/api/travis-3.sh 2
+    sh ./server/tests/api/ci-3.sh 2
 elif [ "$1" = "api-4" ]; then
     npm run build:server
-    sh ./server/tests/api/travis-4.sh 2
+    sh ./server/tests/api/ci-4.sh 2
 elif [ "$1" = "lint" ]; then
     npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
 

scripts/clean/server/test.sh

@@ -5,7 +5,7 @@ set -eu
 recreateDB () {
   dbname="peertube_test$1"
 
-  dropdb --if-exists "$dbname"
+  dropdb --if-exists "$dbname" 2>&1
 
   createdb -O peertube "$dbname"
   psql -c "CREATE EXTENSION pg_trgm;" "$dbname" &
@@ -18,10 +18,15 @@ removeFiles () {
 
 dropRedis () {
   port=$((9000+$1))
+  host="localhost"
 
-  redis-cli KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
-  redis-cli KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
-  redis-cli KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+  if [ ! -z ${GITLAB_CI+x} ]; then
+    host="redis"
+  fi
+
+  redis-cli -h "$host" KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
+  redis-cli -h "$host" KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
+  redis-cli -h "$host" KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
 }
 
 seq=$(seq 1 6)

scripts/nightly.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -eu
+
+shutdown() {
+  # Get our process group id
+  # shellcheck disable=SC2009
+  PGID=$(ps -o pgid= $$ | grep -o "[0-9]*")
+
+  # Kill it in a new new process group
+  setsid kill -- -"$PGID"
+  exit 0
+}
+
+trap "shutdown" SIGINT SIGTERM
+
+today=$(date '+%F')
+directory_name="peertube-nightly-$today"
+tar_name="peertube-nightly-$today.tar.xz"
+
+npm run build
+
+# Creating the archives
+(
+  # local variables
+  directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \
+                          "$directory_name/LICENSE" "$directory_name/README.md" \
+                          "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \
+                          "$directory_name/client/package.json" "$directory_name/config" \
+                          "$directory_name/dist" "$directory_name/package.json" \
+                          "$directory_name/scripts" "$directory_name/support" \
+                          "$directory_name/tsconfig.json" "$directory_name/yarn.lock")
+
+  # temporary setup
+  cd ..
+  ln -s "PeerTube" "$directory_name"
+
+  XZ_OPT=-e9 tar cfJ "PeerTube/$tar_name" "${directories_to_archive[@]}"
+
+  # temporary setup destruction
+  rm "$directory_name"
+)

scripts/test.sh

@@ -5,6 +5,6 @@ set -eu
 npm run build:server
 npm run setup:cli
 
-npm run travis -- lint
+npm run ci -- lint
 
 mocha --exit --require ts-node/register/type-check --bail server/tests/index.ts

server/controllers/api/videos/abuse.ts

@@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
 
     // We send the video abuse to the origin server
     if (videoInstance.isOwned() === false) {
-      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
+      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
     }
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
 
     return videoAbuseInstance
   })
 
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
+
   logger.info('Abuse report for video %s created.', videoInstance.name)
 
   return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()

server/helpers/database-utils.ts

@@ -2,6 +2,7 @@ import * as retry from 'async/retry'
 import * as Bluebird from 'bluebird'
 import { Model } from 'sequelize-typescript'
 import { logger } from './logger'
+import { Transaction } from 'sequelize'
 
 function retryTransactionWrapper <T, A, B, C> (
   functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
@@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
   })
 }
 
+function afterCommitIfTransaction (t: Transaction, fn: Function) {
+  if (t) return t.afterCommit(() => fn())
+
+  return fn()
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   resetSequelizeInstance,
   retryTransactionWrapper,
   transactionRetryer,
-  updateInstanceWithAnother
+  updateInstanceWithAnother,
+  afterCommitIfTransaction
 }

server/lib/activitypub/process/process-flag.ts

@@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
 
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
 
-  return sequelizeTypescript.transaction(async t => {
+  const videoAbuse = await sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {
       reporterAccountId: account.id,
       reason: flag.content,
@@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
     const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
     videoAbuseInstance.Video = video
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     logger.info('Remote abuse for video uuid %s created', flag.object)
+
+    return videoAbuseInstance
   })
+
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
 }

server/lib/activitypub/send/send-create.ts

@@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc
 import { VideoPlaylistModel } from '../../../models/video/video-playlist'
 import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
 import { getServerActor } from '../../../helpers/utils'
+import * as Bluebird from 'bluebird'
 
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
+  await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
@@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
   if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {

server/lib/activitypub/send/send-delete.ts

@@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
+  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(activity, byActor, [ byActor ], t)
@@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
   if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {