More in depth refactoring of providers and modules

src/cmd/add-module/main.go

@@ -41,6 +41,8 @@ func main() {
 	}
 	ghClient := github.NewClient(ctx, logger, token)
 
+	storage := module.NewStorage(*moduleDataDir, logger, ghClient)
+
 	output := Output{}
 
 	err = func() error {
@@ -51,52 +53,44 @@ func main() {
 			return fmt.Errorf("Invalid repository name: %s", *repository)
 		}
 
-		submitted := module.Module{
+		submitted := storage.Create(module.Identifier{
 			Namespace:    match[re.SubexpIndex("Namespace")],
 			Name:         match[re.SubexpIndex("Name")],
 			TargetSystem: match[re.SubexpIndex("Target")],
-			Directory:    *moduleDataDir,
-			Logger:       logger,
-			Github:       ghClient,
-		}
+		})
 
-		_, err = regaddr.ParseModuleSource(fmt.Sprintf("%s/%s/%s", submitted.Namespace, submitted.Name, submitted.TargetSystem))
+		_, err = regaddr.ParseModuleSource(submitted.String())
 		if err != nil {
 			return err
 		}
 
-		modules, err := module.ListModules(*moduleDataDir, "", logger, ghClient)
+		modules, err := storage.List()
 		if err != nil {
 			return err
 		}
 		for _, p := range modules {
-			if strings.ToLower(p.RepositoryURL()) == strings.ToLower(submitted.RepositoryURL()) {
-				return fmt.Errorf("Repository already exists in the registry, %s", p.RepositoryURL())
+			if strings.ToLower(p.String()) == strings.ToLower(submitted.String()) {
+				return fmt.Errorf("Repository already exists in the registry, %s", p.String())
 			}
 		}
 
-		err = submitted.WriteMetadata(module.Metadata{})
+		err = submitted.UpdateMetadata()
 		if err != nil {
 			return fmt.Errorf("An unexpected error occured: %w", err)
 		}
-
-		err = submitted.UpdateMetadataFile()
-		if err != nil {
-			return fmt.Errorf("An unexpected error occured: %w", err)
+		if len(submitted.Versions) == 0 {
+			return fmt.Errorf("No versions detected for repository %s", submitted.Repository.URL())
 		}
 
-		meta, err := submitted.ReadMetadata()
+		err = storage.Save(submitted)
 		if err != nil {
-			return fmt.Errorf("An unexpected error occured: %w", err)
-		}
-		if len(meta.Versions) == 0 {
-			return fmt.Errorf("No versions detected for repository %s", submitted.RepositoryURL())
+			return err
 		}
 
 		output.Namespace = submitted.Namespace
 		output.Name = submitted.Name
 		output.Target = submitted.TargetSystem
-		output.File = submitted.MetadataPath()
+		output.File = storage.Path(submitted.Identifier)
 		return nil
 	}()
 

src/cmd/add-provider/main.go

@@ -40,6 +40,8 @@ func main() {
 	}
 	ghClient := github.NewClient(ctx, logger, token)
 
+	storage := provider.NewStorage(*providerDataDir, logger, ghClient)
+
 	output := Output{}
 
 	err = func() error {
@@ -50,50 +52,43 @@ func main() {
 			return fmt.Errorf("Invalid repository name: %s", *repository)
 		}
 
-		submitted := provider.Provider{
+		submitted := storage.Create(provider.Identifier{
 			Namespace:    match[re.SubexpIndex("Namespace")],
 			ProviderName: match[re.SubexpIndex("Name")],
-			Directory:    *providerDataDir,
-			Logger:       logger,
-			Github:       ghClient,
-		}
+		})
 
-		_, err = regaddr.ParseProviderSource(fmt.Sprintf("%s/%s", submitted.Namespace, submitted.ProviderName))
+		_, err = regaddr.ParseProviderSource(submitted.String())
 		if err != nil {
 			return err
 		}
 
-		providers, err := provider.ListProviders(*providerDataDir, "", logger, ghClient)
+		providers, err := storage.List()
 		if err != nil {
 			return err
 		}
 		for _, p := range providers {
-			if strings.ToLower(p.RepositoryURL()) == strings.ToLower(submitted.RepositoryURL()) {
-				return fmt.Errorf("Repository already exists in the registry, %s", p.RepositoryURL())
+			if strings.ToLower(p.String()) == strings.ToLower(submitted.String()) {
+				return fmt.Errorf("Repository already exists in the registry, %s", p.String())
 			}
 		}
 
-		err = submitted.WriteMetadata(provider.Metadata{})
+		err = submitted.UpdateMetadata()
 		if err != nil {
 			return fmt.Errorf("An unexpected error occured: %w", err)
 		}
 
-		err = submitted.UpdateMetadataFile()
-		if err != nil {
-			return fmt.Errorf("An unexpected error occured: %w", err)
+		if len(submitted.Versions) == 0 {
+			return fmt.Errorf("No versions detected for repository %s", submitted.Repository.URL())
 		}
 
-		meta, err := submitted.ReadMetadata()
+		err = storage.Save(submitted)
 		if err != nil {
 			return fmt.Errorf("An unexpected error occured: %w", err)
 		}
-		if len(meta.Versions) == 0 {
-			return fmt.Errorf("No versions detected for repository %s", submitted.RepositoryURL())
-		}
 
 		output.Namespace = submitted.Namespace
 		output.Name = submitted.ProviderName
-		output.File = submitted.MetadataPath()
+		output.File = storage.Path(submitted.Identifier)
 		return nil
 	}()
 

src/cmd/bump-versions/main.go

@@ -30,30 +30,59 @@ func main() {
 	}
 	ghClient := github.NewClient(ctx, logger, token)
 
-	modules, err := module.ListModules(*moduleDataDir, *moduleNamespace, logger, ghClient)
+	modStorage := module.NewStorage(*moduleDataDir, logger, ghClient)
+	modules, err := modStorage.List()
 	if err != nil {
 		logger.Error("Failed to list modules", slog.Any("err", err))
 		os.Exit(1)
 	}
 
-	err = modules.Parallel(200, func(m module.Module) error {
-		return m.UpdateMetadataFile()
+	errs := modules.ParallelForEach(func(id module.Identifier) error {
+		if *moduleNamespace != "" && *moduleNamespace != id.Namespace {
+			return nil
+		}
+
+		m, err := modStorage.Load(id)
+		if err != nil {
+			return err
+		}
+		err = m.UpdateMetadata()
+		if err != nil {
+			return err
+		}
+		return modStorage.Save(m)
 	})
-	if err != nil {
-		logger.Error(err.Error())
-		os.Exit(1)
-	}
 
-	providers, err := provider.ListProviders(*providerDataDir, *providerNamespace, logger, ghClient)
-	if err != nil {
-		logger.Error("Failed to list providers", slog.Any("err", err))
+	if len(errs) != 0 {
+		logger.Error("Errors occured while processing modules")
+		for _, err := range errs {
+			logger.Error(err.Error())
+		}
 		os.Exit(1)
 	}
-	err = providers.Parallel(200, func(p provider.Provider) error {
-		return p.UpdateMetadataFile()
+
+	provStorage := provider.NewStorage(*providerDataDir, logger, ghClient)
+	providers, err := provStorage.List()
+	errs = providers.ParallelForEach(func(id provider.Identifier) error {
+		if *providerNamespace != "" && *providerNamespace != id.Namespace {
+			return nil
+		}
+
+		p, err := provStorage.Load(id)
+		if err != nil {
+			return err
+		}
+		err = p.UpdateMetadata()
+		if err != nil {
+			return err
+		}
+		return provStorage.Save(p)
 	})
-	if err != nil {
-		logger.Error(err.Error())
+	if len(errs) != 0 {
+		logger.Error("Errors occured while processing providers")
+		for _, err := range errs {
+			logger.Error(err.Error())
+		}
 		os.Exit(1)
 	}
 

src/cmd/generate-v1/main.go

@@ -35,37 +35,53 @@ func main() {
 		os.Exit(1)
 	}
 
-	modules, err := module.ListModules(*moduleDataDir, *moduleNamespace, logger, ghClient)
+	modStorage := module.NewStorage(*moduleDataDir, logger, ghClient)
+	modules, err := modStorage.List()
 	if err != nil {
 		logger.Error("Failed to list modules", slog.Any("err", err))
 		os.Exit(1)
 	}
-	err = modules.Parallel(20, func(m module.Module) error {
-		g, err := v1api.NewModuleGenerator(m, *destinationDir)
+	errs := modules.ParallelForEach(func(id module.Identifier) error {
+		if *moduleNamespace != "" && *moduleNamespace != id.Namespace {
+			return nil
+		}
+
+		m, err := modStorage.Load(id)
 		if err != nil {
 			return err
 		}
-		return g.Generate()
+
+		return v1api.NewModuleGenerator(m, *destinationDir).Generate()
 	})
 	if err != nil {
-		logger.Error(err.Error())
+		logger.Error("Errors occured while processing modules")
+		for _, err := range errs {
+			logger.Error(err.Error())
+		}
 		os.Exit(1)
 	}
 
-	providers, err := provider.ListProviders(*providerDataDir, *providerNamespace, logger, ghClient)
+	provStorage := provider.NewStorage(*providerDataDir, logger, ghClient)
+	providers, err := provStorage.List()
 	if err != nil {
 		logger.Error("Failed to list providers", slog.Any("err", err))
 		os.Exit(1)
 	}
-	err = providers.Parallel(20, func(p provider.Provider) error {
-		g, err := v1api.NewProviderGenerator(p, *destinationDir, *keyDataDir)
+	errs = providers.ParallelForEach(func(id provider.Identifier) error {
+		if *providerNamespace != "" && *providerNamespace != id.Namespace {
+			return nil
+		}
+		p, err := provStorage.Load(id)
 		if err != nil {
 			return err
 		}
-		return g.Generate()
+		return v1api.NewProviderGenerator(p, *destinationDir, *keyDataDir).Generate()
 	})
-	if err != nil {
-		logger.Error(err.Error())
+	if len(errs) != 0 {
+		logger.Error("Errors occured while processing providers")
+		for _, err := range errs {
+			logger.Error(err.Error())
+		}
 		os.Exit(1)
 	}
 

src/internal/base/fs.go

@@ -0,0 +1,39 @@
+package base
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/opentofu/registry-stable/internal/files"
+)
+
+type FileSystem struct {
+	Directory string
+}
+
+func (fs FileSystem) ReadJSONFrom(path string, ref any) error {
+	data, err := os.ReadFile(filepath.Join(fs.Directory, path))
+	if err != nil {
+		return err
+	}
+
+	err = json.Unmarshal(data, ref)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fs FileSystem) WriteJSONInto(path string, data any) error {
+	return files.SafeWriteObjectToJSONFile(filepath.Join(fs.Directory, path), data)
+}
+
+func (fs FileSystem) List() ([]string, error) {
+	paths := make([]string, 0)
+	err := filepath.Walk(fs.Directory, func(path string, info os.FileInfo, err error) error {
+		paths = append(paths, path)
+		return err
+	})
+	return paths, err
+}

src/internal/github/assets.go

@@ -7,21 +7,28 @@ import (
 	"net/http"
 )
 
-// DownloadAssetContents downloads the contents of the asset at the given URL and returns it directly
-func (c Client) DownloadAssetContents(downloadURL string) ([]byte, error) {
-	done := c.assetThrottle()
+func (r Repository) ReleaseAssetURL(release string, asset string) string {
+	return fmt.Sprintf("%s/releases/download/%s/%s", r.URL(), release, asset)
+}
+
+// GetReleaseAsset downloads the contents of the asset and returns it directly
+func (r Repository) GetReleaseAsset(release string, asset string) ([]byte, error) {
+	done := r.client.assetThrottle()
 	defer done()
 
-	logger := c.log.With(slog.String("url", downloadURL))
+	downloadURL := r.ReleaseAssetURL(release, asset)
+
+	logger := r.log.With(slog.String("url", downloadURL))
 	logger.Info("Downloading asset")
 
-	resp, err := c.httpClient.Get(downloadURL)
+	resp, err := r.client.httpClient.Get(downloadURL)
 	if err != nil {
 		return nil, fmt.Errorf("error downloading asset %s: %w", downloadURL, err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusNotFound {
+		// TODO specific error instead of nil
 		logger.Warn("asset not found")
 		return nil, nil
 	}