Merge pull request #4 from OkieOth/recover_for_all

Recover for all

.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish docker image
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: build image
+        run: ./buildImage.sh
+      - name: Login to GitHub Registry
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u okieoth --password-stdin
+      - name: Push the Docker image
+        run: docker push ghcr.io/okieoth/mschemaguesser:`cat version.txt | grep -P '\d+\.\d+\.\d+'`
+      - name: Push the latest Docker image
+        run: docker push ghcr.io/okieoth/mschemaguesser
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          body_path: CHANGELOG.md
+          draft: false
+          prerelease: false

.gitignore

@@ -22,4 +22,5 @@ go.work
 
 tmp.*
 test.*
-tmp
+tmp
+mschemaguesser

CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+* main functionality is implemented
+* CI releases docker image on github

Dockerfile.release

@@ -0,0 +1,23 @@
+FROM golang:1.21 AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY cmd ./cmd
+COPY internal ./internal
+COPY resources ./resources
+
+RUN go test -v --skip "_IT" ./...
+
+RUN CGO_ENABLED=0 GOOS=linux go build -o mschemaguesser cmd/schemaguesser/main.go
+
+FROM alpine:latest
+
+WORKDIR /app
+
+COPY --from=builder /app/mschemaguesser .
+COPY --from=builder /app/resources/json_schema.tmpl ./resources/json_schema.tmpl
+
+ENTRYPOINT [ "./mschemaguesser" ]

README.md

@@ -1,15 +1,75 @@
 [![ci](https://github.com/OkieOth/mschemaguesser/actions/workflows/test.yml/badge.svg)](https://github.com/OkieOth/mschemaguesser/actions/workflows/test.yml)
-![WIP](https://img.shields.io/badge/work%20in%20progress-red)
 
 # TL;DR;
 
 Tool to guess storage models from mongodb
 
-```bash
-# show help
-go run cmd/schemaguesser/main.go help
+![Basic usage](img/usage_help.png)
+
+# Usage
 
+If you have a mongodb installation by hand than you can use this. In
+case you don't the repo contained docker compose stack is a good
+starting point.
+
+```bash
 # start test environment with Mongodb
 ./docker/bin/compose_env.sh start
+```
+
+The default connection string is
+`mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/admin`
+
+It can be customized with the following environment variables:
+* MONGO_USER - User to authenticate, if not set then `admin`
+* MONGO_PASSWORD - Password to authenticate, if not set then `secretpassword`
+* MONGO_HOST - IP address or server name to mongodb, if not set then `localhost`
+* MONGO_PORT - port that is used, if not set then `27017`
+
+In cases where it isn't enough to customize the env variables the commandline
+switch `--con_str` can be used to provide a suitable connection string to the
+program.
+
+
+## Native
+```bash
+go build -o mschemaguesser cmd/schemaguesser/main.go
+
+# print usage
+./mschemaguesser --help
+
+# query all existing databases
+# in general MONGO_HOST can be skipped in case of localhost connections
+MONGO_HOST=localhost ./mschemaguesser list databases
+
+# export the schemas of all databases and their collections
+# in general MONGO_HOST can be skipped in case of localhost connections
+MONGO_HOST=localhost ./mschemaguesser schema --database all --output /tmp
+```
+
+## Docker
+
+```bash
+# print usage
+docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \
+     -e MONGO_HOST=192.168.178.52 \
+     ghcr.io/okieoth/mschemaguesser --help
+
+# query all existing databases
+docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \
+     -e MONGO_HOST=192.168.178.52 \
+     ghcr.io/okieoth/mschemaguesser list databases
+
+# export the schemas of all databases and their collections
+docker run -u $(id -u ${USER}):$(id -g ${USER}) --rm \
+     -v $(pwd)/tmp:/output_dir \
+     -e MONGO_HOST=192.168.178.52 \
+     ghcr.io/okieoth/mschemaguesser schema \
+     --database all --output /output_dir
+```
+
+```bash
+# show help
+go run cmd/schemaguesser/main.go help
 
 ```

buildImage.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+scriptPos=${0%/*}
+
+
+imageBase=ghcr.io/okieoth/mschemaguesser
+imageTag=`cat $scriptPos/version.txt | grep -P '\d+\.\d+\.\d+'`
+
+imageName="$imageBase:$imageTag"
+
+echo "I am going to create: $imageName"
+
+pushd "$scriptPos" > /dev/null
+        if docker build -f Dockerfile.release -t $imageName .
+    then
+        docker tag $imageName $imageBase
+        echo -en "\033[1;34m  image created: $imageName, $imageBase \033[0m\n"
+    else
+        echo -en "\033[1;31m  error while create image: $imageName \033[0m\n"
+    fi
+popd > /dev/null

cmd/schemaguesser/cmd/schema.go

@@ -28,7 +28,7 @@ var schemaCmd = &cobra.Command{
 			if colName == "all" {
 				printSchemasForAllCollections(dbName)
 			} else {
-				printSchemaForOneCollection(dbName, colName)
+				printSchemaForOneCollection(dbName, colName, false)
 			}
 		}
 
@@ -45,7 +45,14 @@ func init() {
 	schemaCmd.Flags().Int32Var(&itemCount, "item_count", 100, "Number of collection entries used to build the schema")
 }
 
-func printSchemaForOneCollection(dbName string, collName string) {
+func printSchemaForOneCollection(dbName string, collName string, doRecover bool) {
+	defer func() {
+		if doRecover {
+			if r := recover(); r != nil {
+				fmt.Printf("Recovered while handling collection (db: %s, collection: %s): %v", dbName, collName, r)
+			}
+		}
+	}()
 	bsonRaw, err := mongoHelper.QueryCollection(mongoHelper.ConStr, dbName, collName, int(itemCount))
 	if err != nil {
 		msg := fmt.Sprintf("Error while reading data for collection (%s.%s): \n%v\n", dbName, collName, err)
@@ -65,7 +72,7 @@ func printSchemaForOneCollection(dbName string, collName string) {
 func printSchemasForAllCollections(dbName string) {
 	collections := mongoHelper.ReadCollectionsOrPanic(dbName)
 	for _, coll := range *collections {
-		printSchemaForOneCollection(dbName, coll)
+		printSchemaForOneCollection(dbName, coll, true)
 	}
 }
 

internal/pkg/mongoHelper/mongoHelper.go

@@ -3,7 +3,6 @@ package mongoHelper
 import (
 	"context"
 	"fmt"
-	"log"
 	"okieoth/schemaguesser/internal/pkg/utils"
 
 	"go.mongodb.org/mongo-driver/bson"
@@ -38,14 +37,13 @@ func ListDatabases(conStr string) ([]string, error) {
 			return
 		}
 		if err = client.Disconnect(context.Background()); err != nil {
-			log.Fatal(err)
+			println("Error while disconnect: %v", err)
 		}
 	}()
 
 	cursor, err := client.ListDatabases(context.Background(), bson.M{})
 	if err != nil {
-		log.Fatal(err)
-		return ret, err
+		panic(err)
 	}
 
 	for _, db := range cursor.Databases {
@@ -66,15 +64,14 @@ func ListCollections(conStr string, databaseName string) ([]string, error) {
 			return
 		}
 		if err = client.Disconnect(context.Background()); err != nil {
-			log.Fatal(err)
+			println("Error while disconnect: %v", err)
 		}
 	}()
 
 	db := client.Database(databaseName)
 	cursor, err := db.ListCollectionNames(context.Background(), bson.M{})
 	if err != nil {
-		log.Fatal(err)
-		return ret, err
+		panic(err)
 	}
 
 	for _, collName := range cursor {
@@ -95,7 +92,7 @@ func ListIndexes(conStr string, databaseName string, collectionName string) ([]s
 			return
 		}
 		if err = client.Disconnect(context.Background()); err != nil {
-			log.Fatal(err)
+			println("Error while disconnect: %v", err)
 		}
 	}()
 
@@ -105,8 +102,7 @@ func ListIndexes(conStr string, databaseName string, collectionName string) ([]s
 	cursor, err := indexView.List(context.Background())
 
 	if err != nil {
-		log.Fatal(err)
-		return ret, err
+		panic(err)
 	}
 
 	for cursor.Next(context.Background()) {
@@ -129,7 +125,7 @@ func QueryCollection(conStr string, databaseName string, collectionName string,
 			return
 		}
 		if err = client.Disconnect(context.Background()); err != nil {
-			log.Fatal(err)
+			println("Error while disconnect: %v", err)
 		}
 	}()
 
@@ -138,8 +134,7 @@ func QueryCollection(conStr string, databaseName string, collectionName string,
 	cursor, err := collection.Find(context.Background(), bson.M{})
 
 	if err != nil {
-		log.Fatal(err)
-		return ret, err
+		panic(err)
 	}
 
 	i := 0

version.txt

@@ -0,0 +1 @@
+1.0.0