Testrig fixes (#50)

* testrig is runnable again
* little fixes, add some more test models
* address #44

CONTRIBUTING.md

@@ -56,6 +56,42 @@ docker run -d --user postgres --network host sosedoff/pgweb
 
 This will launch a pgweb at `http://localhost:8081`.
 
+### Standalone Testrig
+
+You can also launch a testrig as a standalone server running at localhost, which you can connect to using something like [Pinafore](https://github.com/nolanlawson/pinafore).
+
+To do this, first build the gotosocial binary with `go build ./cmd/gotosocial`.
+
+Then launch a clean Postgres container on localhost:
+
+```bash
+docker run -d --user postgres --network host -e POSTGRES_PASSWORD=postgres postgres
+```
+
+Then, launch the testrig by invoking the binary as follows:
+
+```bash
+./gotosocial --host localhost:8080 testrig start
+```
+
+To run Pinafore locally in dev mode, first clone the Pinafore repository, and run the following command in the cloned directory:
+
+```bash
+yarn run dev
+```
+
+The Pinafore instance will start running on `localhost:4002`.
+
+To connect to the testrig, navigate to `https://localhost:4002` and enter your instance name as `localhost:8080`.
+
+At the login screen, enter the email address `zork@example.org` and password `password`.
+
+Note the following constraints:
+
+- The testrig data will be destroyed when the testrig is destroyed. It does this by dropping all tables in Postgres on shutdown. As such, you should **NEVER RUN THE TESTRIG AGAINST A DATABASE WITH REAL DATA IN IT** because it will be destroyed. Be especially careful if you're forwarding database ports from a remote instance to your local machine, because you can easily and irreversibly nuke that data if you run the testrig against it.
+- If you stop the testrig and start it again, any tokens or applications you created during your tests will also be removed. As such, you need to log out and in again every time you stop/start the rig.
+- The testrig does not make any actual external http calls, so federation will (obviously) not work from a testrig.
+
 ## Running tests
 
 Because the tests use a real Postgres under the hood, you can't run them in parallel, so you need to run tests with the following command:
@@ -98,4 +134,4 @@ Then make sure to run `go fmt ./...` to update whitespace and other opinionated
 
 Right now there's no structure in place for financial compensation for pull requests and code. This is simply because there's no money being made on the project apart from the very small weekly Liberapay donations.
 
-If money starts coming in, I'll start looking at proper financial structures, but for now code is considered to be a donation in itself.
+If money starts coming in, I'll start looking at proper financial structures, but for now code is considered to be a donation in itself.

cmd/gotosocial/main.go

@@ -26,9 +26,9 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account"
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions/server"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions/testrig"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/testrig"
 
 	"github.com/urfave/cli/v2"
 )
@@ -361,19 +361,6 @@ func main() {
 					},
 				},
 			},
-			// {
-			// 	Name:  "db",
-			// 	Usage: "database-related tasks and utils",
-			// 	Subcommands: []*cli.Command{
-			// 		{
-			// 			Name:  "init",
-			// 			Usage: "initialize a database with the required schema for gotosocial; has no effect & is safe to run on an already-initialized db",
-			// 			Action: func(c *cli.Context) error {
-			// 				return runAction(c, db.Initialize)
-			// 			},
-			// 		},
-			// 	},
-			// },
 			{
 				Name:  "testrig",
 				Usage: "gotosocial testrig tasks",
@@ -382,7 +369,7 @@ func main() {
 						Name:  "start",
 						Usage: "start the gotosocial testrig",
 						Action: func(c *cli.Context) error {
-							return runAction(c, testrig.Run)
+							return runAction(c, testrig.Start)
 						},
 					},
 				},

testrig/actions.go → internal/cliactions/testrig/testrig.go

@@ -1,21 +1,3 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
 package testrig
 
 import (
@@ -34,50 +16,72 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/app"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/auth"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/emoji"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/filter"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/list"
 	mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/notification"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/streaming"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
 	"github.com/superseriousbusiness/gotosocial/internal/api/security"
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
-	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
+	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
-// Run creates and starts a gotosocial testrig server
-var Run cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error {
-	c := NewTestConfig()
-	dbService := NewTestDB()
-	federatingDB := NewTestFederatingDB(dbService)
-	router := NewTestRouter()
-	storageBackend := NewTestStorage()
-
-	typeConverter := NewTestTypeConverter(dbService)
-	transportController := NewTestTransportController(NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+// Start creates and starts a gotosocial testrig server
+var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error {
+	c := testrig.NewTestConfig()
+	dbService := testrig.NewTestDB()
+	testrig.StandardDBSetup(dbService)
+	router := testrig.NewTestRouter()
+	storageBackend := testrig.NewTestStorage()
+	testrig.StandardStorageSetup(storageBackend, "./testrig/media")
+
+	// build backend handlers
+	oauthServer := testrig.NewTestOauthServer(dbService)
+	transportController := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
 		r := ioutil.NopCloser(bytes.NewReader([]byte{}))
 		return &http.Response{
 			StatusCode: 200,
 			Body:       r,
 		}, nil
 	}))
-	federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter)
-	processor := NewTestProcessor(dbService, storageBackend, federator)
+	federator := testrig.NewTestFederator(dbService, transportController)
+
+	processor := testrig.NewTestProcessor(dbService, storageBackend, federator)
 	if err := processor.Start(); err != nil {
 		return fmt.Errorf("error starting processor: %s", err)
 	}
 
-	StandardDBSetup(dbService)
-	StandardStorageSetup(storageBackend, "./testrig/media")
-
 	// build client api modules
-	authModule := auth.New(c, dbService, NewTestOauthServer(dbService), log)
+	authModule := auth.New(c, dbService, oauthServer, log)
 	accountModule := account.New(c, processor, log)
+	instanceModule := instance.New(c, processor, log)
 	appsModule := app.New(c, processor, log)
+	followRequestsModule := followrequest.New(c, processor, log)
+	webfingerModule := webfinger.New(c, processor, log)
+	usersModule := user.New(c, processor, log)
+	timelineModule := timeline.New(c, processor, log)
+	notificationModule := notification.New(c, processor, log)
+	searchModule := search.New(c, processor, log)
+	filtersModule := filter.New(c, processor, log)
+	emojiModule := emoji.New(c, processor, log)
+	listsModule := list.New(c, processor, log)
 	mm := mediaModule.New(c, processor, log)
 	fileServerModule := fileserver.New(c, processor, log)
 	adminModule := admin.New(c, processor, log)
 	statusModule := status.New(c, processor, log)
 	securityModule := security.New(c, log)
+	streamingModule := streaming.New(c, processor, log)
 
 	apis := []api.ClientModule{
 		// modules with middleware go first
@@ -86,11 +90,22 @@ var Run cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *
 
 		// now everything else
 		accountModule,
+		instanceModule,
 		appsModule,
+		followRequestsModule,
 		mm,
 		fileServerModule,
 		adminModule,
 		statusModule,
+		webfingerModule,
+		usersModule,
+		timelineModule,
+		notificationModule,
+		searchModule,
+		filtersModule,
+		emojiModule,
+		listsModule,
+		streamingModule,
 	}
 
 	for _, m := range apis {
@@ -114,8 +129,8 @@ var Run cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *
 	sig := <-sigs
 	log.Infof("received signal %s, shutting down", sig)
 
-	StandardDBTeardown(dbService)
-	StandardStorageTeardown(storageBackend)
+	testrig.StandardDBTeardown(dbService)
+	testrig.StandardStorageTeardown(storageBackend)
 
 	// close down all running services in order
 	if err := gts.Stop(ctx); err != nil {

internal/timeline/prepare.go

@@ -1,6 +1,7 @@
 package timeline
 
 import (
+	"container/list"
 	"errors"
 	"fmt"
 
@@ -31,6 +32,17 @@ func (t *timeline) PrepareBehind(statusID string, amount int) error {
 	t.Lock()
 	defer t.Unlock()
 
+	// lazily initialize prepared posts if it hasn't been done already
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+		t.preparedPosts.data.Init()
+	}
+
+	// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
+	if t.postIndex.data == nil {
+		return nil
+	}
+
 	var prepared int
 	var preparing bool
 prepareloop:
@@ -72,6 +84,17 @@ func (t *timeline) PrepareBefore(statusID string, include bool, amount int) erro
 	t.Lock()
 	defer t.Unlock()
 
+	// lazily initialize prepared posts if it hasn't been done already
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+		t.preparedPosts.data.Init()
+	}
+
+	// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
+	if t.postIndex.data == nil {
+		return nil
+	}
+
 	var prepared int
 	var preparing bool
 prepareloop:
@@ -116,11 +139,24 @@ func (t *timeline) PrepareFromTop(amount int) error {
 	t.Lock()
 	defer t.Unlock()
 
-	t.preparedPosts.data.Init()
+	// lazily initialize prepared posts if it hasn't been done already
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+		t.preparedPosts.data.Init()
+	}
+
+	// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
+	if t.postIndex.data == nil {
+		return nil
+	}
 
 	var prepared int
 prepareloop:
 	for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+		if e == nil {
+			continue
+		}
+
 		entry, ok := e.Value.(*postIndexEntry)
 		if !ok {
 			return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")

testrig/db.go

@@ -45,6 +45,8 @@ var testModels []interface{} = []interface{}{
 	&gtsmodel.Tag{},
 	&gtsmodel.User{},
 	&gtsmodel.Emoji{},
+	&gtsmodel.Instance{},
+	&gtsmodel.Notification{},
 	&oauth.Token{},
 	&oauth.Client{},
 }
@@ -129,9 +131,25 @@ func StandardDBSetup(db db.DB) {
 		}
 	}
 
+	for _, v := range NewTestFollows() {
+		if err := db.Put(v); err != nil {
+			panic(err)
+		}
+	}
+
+	for _, v := range NewTestNotifications() {
+		if err := db.Put(v); err != nil {
+			panic(err)
+		}
+	}
+
 	if err := db.CreateInstanceAccount(); err != nil {
 		panic(err)
 	}
+
+	if err := db.CreateInstanceInstance(); err != nil {
+		panic(err)
+	}
 }
 
 // StandardDBTeardown drops all the standard testing tables/models from the database to ensure it's clean for the next test.