Fix, use conceptType not content type, search should be fixed as well

cmd/content-rw-elasticsearch/main.go

@@ -5,13 +5,13 @@ import (
 	"os"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/es"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/health"
-	pkghttp "github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/http"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/message"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/health"
+	pkghttp "github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/http"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/message"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/kafka-client-go/v2"
 	cli "github.com/jawher/mow.cli"

configs/migrate.sh

@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
+
+echo "Don't run this script, is not totally compatible anymore, but is useful as documentation for the relevant changes"
+exit 5
 index_name="combinedpostpublicationevents"
 file_path=$(dirname "$0")/referenceSchema.json
 # This make sure that the script is apply to the same file that the script was created for
 # due to poor portability
 if [ $(git log --follow -1 --pretty=format:"%h"  -- "$file_path") -ne 9781498 ]; then
     echo "This script was created to migrate $file_path from revision 9781498"
-    exit -1
+    exit 1
 fi
 pushd $(dirname "$0"); git restore "$file_path"; popd
 

go.mod

@@ -1,4 +1,4 @@
-module github.com/Financial-Times/content-rw-elasticsearch/v7
+module github.com/Financial-Times/content-rw-elasticsearch/v2
 
 go 1.17
 
@@ -10,11 +10,11 @@ require (
 	github.com/Financial-Times/transactionid-utils-go v0.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/jawher/mow.cli v1.0.4
+	github.com/olivere/elastic/v7 v7.0.29
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
 	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9
 	github.com/spf13/viper v1.9.0
 	github.com/stretchr/testify v1.7.0
-	github.com/olivere/elastic/v7 v7.0.29
 )
 
 require (
@@ -26,6 +26,7 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/hashicorp/go-version v1.0.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@@ -47,6 +48,7 @@ require (
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect
 	golang.org/x/net v0.0.0-20210917221730-978cfadd31cf // indirect
@@ -55,6 +57,4 @@ require (
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	github.com/hashicorp/go-version v1.0.0 // indirect
-	github.com/stretchr/objx v0.1.1 // indirect
 )

pkg/config/config.go

@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 	"github.com/spf13/viper"
 )
 

pkg/es/service.go

@@ -8,7 +8,7 @@ import (
 	"reflect"
 	"sync"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
 
 	"github.com/olivere/elastic/v7"
 )

pkg/health/healthcheck.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/es"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/message"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/message"
 	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/service-status-go/gtg"

pkg/mapper/mapper.go

@@ -7,10 +7,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/html"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/html"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 	"github.com/Financial-Times/go-logger/v2"
 )
 
@@ -55,7 +55,7 @@ func (h *Handler) ToIndexModel(enrichedContent schema.EnrichedContent, contentTy
 	}
 	h.populateContentRelatedFields(&model, enrichedContent, contentType, tid)
 	model.Type = new(string)
-	*model.Type = contentType
+	*model.Type = h.Config.ESContentTypeMetadataMap.Get(contentType).Collection
 
 	annotations, concepts, err := h.prepareAnnotationsWithConcepts(&enrichedContent, tid)
 	log := h.log.WithTransactionID(tid).WithUUID(enrichedContent.UUID)

pkg/mapper/mapper_test.go

@@ -9,10 +9,10 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/schema"
-	tst "github.com/Financial-Times/content-rw-elasticsearch/v7/test"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
+	tst "github.com/Financial-Times/content-rw-elasticsearch/v2/test"
 	"github.com/Financial-Times/go-logger/v2"
 )
 

pkg/message/message_handler.go

@@ -7,10 +7,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/es"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/schema"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/kafka-client-go/v2"
 	transactionid "github.com/Financial-Times/transactionid-utils-go"

pkg/message/message_handler_test.go

@@ -9,12 +9,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/concept"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/es"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/mapper"
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/schema"
-	tst "github.com/Financial-Times/content-rw-elasticsearch/v7/test"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/concept"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/es"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/mapper"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/schema"
+	tst "github.com/Financial-Times/content-rw-elasticsearch/v2/test"
 	"github.com/Financial-Times/go-logger/v2"
 	"github.com/Financial-Times/kafka-client-go/v2"
 	"github.com/olivere/elastic/v7"

test/helper.go

@@ -8,7 +8,7 @@ import (
 	"path"
 	"strings"
 
-	"github.com/Financial-Times/content-rw-elasticsearch/v7/pkg/config"
+	"github.com/Financial-Times/content-rw-elasticsearch/v2/pkg/config"
 )
 
 func getResourceFilePath(resourceFilePath string) string {

test/testdata/exampleElasticModel.json

@@ -1,5 +1,5 @@
 {
-  "type": "article",
+  "type": "FTCom",
   "uid": "aae9611e-f66c-4fe4-a6c6-2e2bdea69060",
   "last_metadata_publish": null,
   "index_date": null,

test/testdata/testElasticModel1.json

@@ -1,5 +1,5 @@
 {
-  "type": "article",
+  "type": "FTCom",
   "uid": "35ebcdf5-54b2-4834-8309-ba5ae3d5dd15",
   "last_metadata_publish": null,
   "index_date": null,

test/testdata/testElasticModel2.json

@@ -1,5 +1,5 @@
 {
-  "type": "article",
+  "type": "FTCom",
   "uid": "62c53650-68d6-4ed9-9932-e76c53bebd35",
   "last_metadata_publish": null,
   "index_date": null,

test/testdata/testElasticModel4.json

@@ -1,5 +1,5 @@
 {
-  "type": "video",
+  "type": "FTVideos",
   "uid": "0fb3501b-16bf-4cd3-845e-d047ab6d8109",
   "last_metadata_publish": null,
   "index_date": null,