diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..cdfd209 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,33 @@ +name: Publish docker image + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: build image + run: ./buildImage.sh + - name: Login to GitHub Registry + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u okieoth --password-stdin + - name: Push the Docker image + run: docker push ghcr.io/okieoth/mschemaguesser:`cat version.txt | grep -P '\d+\.\d+\.\d+'` + - name: Push the latest Docker image + run: docker push ghcr.io/okieoth/mschemaguesser + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body_path: CHANGELOG.md + draft: false + prerelease: false diff --git a/.gitignore b/.gitignore index 1404061..1f092ac 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ go.work tmp.* test.* -tmp \ No newline at end of file +tmp +mschemaguesser diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..00c65b5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# 1.0.0 +* main functionality is implemented +* CI releases docker image on github \ No newline at end of file diff --git a/Dockerfile.release b/Dockerfile.release new file mode 100644 index 0000000..1b0d6f0 --- /dev/null +++ b/Dockerfile.release @@ -0,0 +1,23 @@ +FROM golang:1.21 AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd ./cmd +COPY internal ./internal +COPY resources ./resources + +RUN go test -v --skip "_IT" ./... + +RUN CGO_ENABLED=0 GOOS=linux go build -o mschemaguesser cmd/schemaguesser/main.go + +FROM alpine:latest + +WORKDIR /app + +COPY --from=builder /app/mschemaguesser . +COPY --from=builder /app/resources/json_schema.tmpl ./resources/json_schema.tmpl + +ENTRYPOINT [ "./mschemaguesser" ] \ No newline at end of file diff --git a/README.md b/README.md index 72706c9..d8922d4 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,75 @@ [![ci](https://github.com/OkieOth/mschemaguesser/actions/workflows/test.yml/badge.svg)](https://github.com/OkieOth/mschemaguesser/actions/workflows/test.yml) -![WIP](https://img.shields.io/badge/work%20in%20progress-red) # TL;DR; Tool to guess storage models from mongodb -```bash -# show help -go run cmd/schemaguesser/main.go help +![Basic usage](img/usage_help.png) + +# Usage +If you have a mongodb installation by hand than you can use this. In +case you don't the repo contained docker compose stack is a good +starting point. + +```bash # start test environment with Mongodb ./docker/bin/compose_env.sh start +``` + +The default connection string is +`mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/admin` + +It can be customized with the following environment variables: +* MONGO_USER - User to authenticate, if not set then `admin` +* MONGO_PASSWORD - Password to authenticate, if not set then `secretpassword` +* MONGO_HOST - IP address or server name to mongodb, if not set then `localhost` +* MONGO_PORT - port that is used, if not set then `27017` + +In cases where it isn't enough to customize the env variables the commandline +switch `--con_str` can be used to provide a suitable connection string to the +program. + + +## Native +```bash +go build -o mschemaguesser cmd/schemaguesser/main.go + +# print usage +./mschemaguesser --help + +# query all existing databases +# in general MONGO_HOST can be skipped in case of localhost connections +MONGO_HOST=localhost ./mschemaguesser list databases + +# export the schemas of all databases and their collections +# in general MONGO_HOST can be skipped in case of localhost connections +MONGO_HOST=localhost ./mschemaguesser schema --database all --output /tmp +``` + +## Docker + +```bash +# print usage +docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \ + -e MONGO_HOST=192.168.178.52 \ + ghcr.io/okieoth/mschemaguesser --help + +# query all existing databases +docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \ + -e MONGO_HOST=192.168.178.52 \ + ghcr.io/okieoth/mschemaguesser list databases + +# export the schemas of all databases and their collections +docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \ + -v $(pwd)/tmp:/output_dir \ + -e MONGO_HOST=192.168.178.52 \ + ghcr.io/okieoth/mschemaguesser schema \ + --database all --output /output_dir +``` + +```bash +# show help +go run cmd/schemaguesser/main.go help ``` diff --git a/buildImage.sh b/buildImage.sh new file mode 100755 index 0000000..64c1d44 --- /dev/null +++ b/buildImage.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +scriptPos=${0%/*} + + +imageBase=ghcr.io/okieoth/mschemaguesser +imageTag=`cat $scriptPos/version.txt | grep -P '\d+\.\d+\.\d+'` + +imageName="$imageBase:$imageTag" + +echo "I am going to create: $imageName" + +pushd "$scriptPos" > /dev/null + if docker build -f Dockerfile.release -t $imageName . + then + docker tag $imageName $imageBase + echo -en "\033[1;34m image created: $imageName, $imageBase \033[0m\n" + else + echo -en "\033[1;31m error while create image: $imageName \033[0m\n" + fi +popd > /dev/null \ No newline at end of file diff --git a/cmd/schemaguesser/cmd/schema.go b/cmd/schemaguesser/cmd/schema.go index 9487466..571109b 100644 --- a/cmd/schemaguesser/cmd/schema.go +++ b/cmd/schemaguesser/cmd/schema.go @@ -28,7 +28,7 @@ var schemaCmd = &cobra.Command{ if colName == "all" { printSchemasForAllCollections(dbName) } else { - printSchemaForOneCollection(dbName, colName) + printSchemaForOneCollection(dbName, colName, false) } } @@ -45,7 +45,14 @@ func init() { schemaCmd.Flags().Int32Var(&itemCount, "item_count", 100, "Number of collection entries used to build the schema") } -func printSchemaForOneCollection(dbName string, collName string) { +func printSchemaForOneCollection(dbName string, collName string, doRecover bool) { + defer func() { + if doRecover { + if r := recover(); r != nil { + fmt.Printf("Recovered while handling collection (db: %s, collection: %s): %v", dbName, collName, r) + } + } + }() bsonRaw, err := mongoHelper.QueryCollection(mongoHelper.ConStr, dbName, collName, int(itemCount)) if err != nil { msg := fmt.Sprintf("Error while reading data for collection (%s.%s): \n%v\n", dbName, collName, err) @@ -65,7 +72,7 @@ func printSchemaForOneCollection(dbName string, collName string) { func printSchemasForAllCollections(dbName string) { collections := mongoHelper.ReadCollectionsOrPanic(dbName) for _, coll := range *collections { - printSchemaForOneCollection(dbName, coll) + printSchemaForOneCollection(dbName, coll, true) } } diff --git a/img/usage_help.png b/img/usage_help.png new file mode 100644 index 0000000..708772d Binary files /dev/null and b/img/usage_help.png differ diff --git a/internal/pkg/mongoHelper/mongoHelper.go b/internal/pkg/mongoHelper/mongoHelper.go index f3e6428..7be8f55 100644 --- a/internal/pkg/mongoHelper/mongoHelper.go +++ b/internal/pkg/mongoHelper/mongoHelper.go @@ -3,7 +3,6 @@ package mongoHelper import ( "context" "fmt" - "log" "okieoth/schemaguesser/internal/pkg/utils" "go.mongodb.org/mongo-driver/bson" @@ -38,14 +37,13 @@ func ListDatabases(conStr string) ([]string, error) { return } if err = client.Disconnect(context.Background()); err != nil { - log.Fatal(err) + println("Error while disconnect: %v", err) } }() cursor, err := client.ListDatabases(context.Background(), bson.M{}) if err != nil { - log.Fatal(err) - return ret, err + panic(err) } for _, db := range cursor.Databases { @@ -66,15 +64,14 @@ func ListCollections(conStr string, databaseName string) ([]string, error) { return } if err = client.Disconnect(context.Background()); err != nil { - log.Fatal(err) + println("Error while disconnect: %v", err) } }() db := client.Database(databaseName) cursor, err := db.ListCollectionNames(context.Background(), bson.M{}) if err != nil { - log.Fatal(err) - return ret, err + panic(err) } for _, collName := range cursor { @@ -95,7 +92,7 @@ func ListIndexes(conStr string, databaseName string, collectionName string) ([]s return } if err = client.Disconnect(context.Background()); err != nil { - log.Fatal(err) + println("Error while disconnect: %v", err) } }() @@ -105,8 +102,7 @@ func ListIndexes(conStr string, databaseName string, collectionName string) ([]s cursor, err := indexView.List(context.Background()) if err != nil { - log.Fatal(err) - return ret, err + panic(err) } for cursor.Next(context.Background()) { @@ -129,7 +125,7 @@ func QueryCollection(conStr string, databaseName string, collectionName string, return } if err = client.Disconnect(context.Background()); err != nil { - log.Fatal(err) + println("Error while disconnect: %v", err) } }() @@ -138,8 +134,7 @@ func QueryCollection(conStr string, databaseName string, collectionName string, cursor, err := collection.Find(context.Background(), bson.M{}) if err != nil { - log.Fatal(err) - return ret, err + panic(err) } i := 0 diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..afaf360 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file