Skip to content

Commit

Permalink
use armtemplate to list all the resources under a group
Browse files Browse the repository at this point in the history
  • Loading branch information
@magodo
magodo committed Aug 23, 2021
1 parent ddc91d6 commit 97e3b25
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 79 deletions.
12 changes: 6 additions & 6 deletions internal/armtemplate/armtemplate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ func NewResourceId(id string) (*ResourceId, error) {
n := []string{}

for i := 0; i < len(segs); i += 2 {
t = append(t, segs[0])
n = append(n, segs[1])
t = append(t, segs[i])
n = append(n, segs[i+1])
}

return &ResourceId{
Expand All @@ -73,12 +73,12 @@ func (res ResourceId) ID(sub, rg string) string {
typeSegs := strings.Split(res.Type, "/")
nameSegs := strings.Split(res.Name, "/")

out := []string{"subscriptions", sub, "resourceGroups", rg}
if len(typeSegs) != 0 {
out := []string{"/subscriptions", sub, "resourceGroups", rg}
if len(typeSegs) != 1 {
if len(typeSegs)-1 != len(nameSegs) {
panic(fmt.Sprintf("The resource of type %q and name %q is not a valid identifier", res.Type, res.Name))
}
out = append(out, typeSegs[0])
out = append(out, "providers", typeSegs[0])
for i := 0; i < len(nameSegs); i++ {
out = append(out, typeSegs[i+1])
out = append(out, nameSegs[i])
Expand Down Expand Up @@ -125,7 +125,7 @@ func (deps *Dependencies) UnmarshalJSON(b []byte) error {

type DependencyInfo map[ResourceId][]ResourceId

func (tpl Template) DependencyInfo(rgName string) DependencyInfo {
func (tpl Template) DependencyInfo() DependencyInfo {
s := map[ResourceId][]ResourceId{}
for _, res := range tpl.Resources {
if len(res.DependsOn) == 0 {
Expand Down
10 changes: 9 additions & 1 deletion internal/armtemplate/armtemplate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func TestDependencyInfo(t *testing.T) {
}

for _, c := range cases {
require.Equal(t, c.expect, c.input.DependencyInfo("rg"), c.name)
require.Equal(t, c.expect, c.input.DependencyInfo(), c.name)
}
}

Expand Down Expand Up @@ -374,6 +374,14 @@ func TestNewResourceId(t *testing.T) {
input: "/subscriptions/1234/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets",
error: true,
},
{
name: "valid subnet id",
input: "/subscriptions/1234/resourcegroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/subnet1",
expect: armtemplate.ResourceId{
Type: "Microsoft.Network/virtualNetworks/subnets",
Name: "vnet1/subnet1",
},
},
}

for _, c := range cases {
Expand Down
119 changes: 50 additions & 69 deletions internal/meta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Meta struct {
workspace string
tf *tfexec.Terraform
auth *Authorizer
armTemplate armtemplate.Template
}

func NewMeta(ctx context.Context, rg string) (*Meta, error) {
Expand Down Expand Up @@ -112,40 +113,53 @@ func (meta *Meta) InitProvider(ctx context.Context) error {
return nil
}

func (meta *Meta) ListResources(ctx context.Context) ([]string, error) {
rgc := meta.auth.NewResourceGroupClient()
resp, err := rgc.Get(ctx, meta.resourceGroup)
func (meta *Meta) ExportArmTemplate(ctx context.Context) error {
client := meta.auth.NewResourceGroupClient()

exportOpt := "SkipAllParameterization"
future, err := client.ExportTemplate(ctx, meta.resourceGroup, resources.ExportTemplateRequest{
ResourcesProperty: &[]string{"*"},
Options: &exportOpt,
})
if err != nil {
return nil, fmt.Errorf("getting resource group %q: %w", meta.resourceGroup, err)
}
if resp.ID == nil || *resp.ID == "" {
return nil, fmt.Errorf("unexpected nil/empty ID for resource group %q", meta.resourceGroup)
return fmt.Errorf("exporting arm template of resource group %s: %w", meta.resourceGroup, err)
}

ids := []string{*resp.ID}
if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return fmt.Errorf("waiting for exporting arm template of resource group %s: %w", meta.resourceGroup, err)
}

// Retrieve the IDs of the embedded resources
rc := meta.auth.NewResourceClient()
results, err := rc.ListByResourceGroupComplete(ctx, meta.resourceGroup, "", "", nil)
result, err := future.Result(client)
if err != nil {
return nil, fmt.Errorf("listing resources in resource group %q: %w", meta.resourceGroup, err)
return fmt.Errorf("getting the arm template of resource group %s: %w", meta.resourceGroup, err)
}
for results.NotDone() {
val := results.Value()
if val.ID != nil && *val.ID != "" {
ids = append(ids, *val.ID)
}

if err := results.NextWithContext(ctx); err != nil {
return nil, fmt.Errorf("retrieving next page of nested resources in resource group %q: %w", meta.resourceGroup, err)
}
// The response has been read into the ".Template" field as an interface, and the reader has been drained.
// As we have defined some (useful) types for the arm template, so we will do a json marshal then unmarshal here
// to convert the ".Template" (interface{}) into that artificial type.
raw, err := json.Marshal(result.Template)
if err != nil {
return fmt.Errorf("marshalling the template: %w", err)
}

return ids, nil
if err := json.Unmarshal(raw, &meta.armTemplate); err != nil {
return fmt.Errorf("unmarshalling the template: %w", err)
}
return nil
}

type ImportList []ImportItem

func (l ImportList) NonSkipped() ImportList {
var out ImportList
for _, item := range l {
if item.Skip {
continue
}
out = append(out, item)
}
return out
}

type ImportItem struct {
ResourceID string
Skip bool
Expand All @@ -160,16 +174,19 @@ func (item *ImportItem) TFAddr() string {
return item.TFResourceType + "." + item.TFResourceName
}

func (meta *Meta) ResolveImportList(ctx context.Context, ids []string) (ImportList, error) {
if len(ids) == 0 {
return nil, nil
func (meta *Meta) ResolveImportList(ctx context.Context) (ImportList, error) {
var ids []string
for _, res := range meta.armTemplate.Resources {
ids = append(ids, res.ID(meta.subscriptionId, meta.resourceGroup))
}
ids = append(ids, armtemplate.ResourceGroupId.ID(meta.subscriptionId, meta.resourceGroup))

// schema, err := meta.tf.ProvidersSchema(ctx)
// if err != nil {
// return nil, fmt.Errorf("getting provider schema: %w", err)
// }
// tfResourceMap := schema.Schemas["registry.terraform.io/hashicorp/azurerm"].ResourceSchemas

tfResourceMap := schema.ProviderSchemaInfo.ResourceSchemas

var list ImportList
Expand Down Expand Up @@ -228,11 +245,9 @@ func (meta *Meta) Import(ctx context.Context, list ImportList) error {
// Generate a temp Terraform config to include the empty template for each resource.
// This is required for the following importing.
cfgFile := filepath.Join(meta.workspace, "main.tf")

var tpls []string
for _, item := range list {
if item.Skip {
continue
}
for _, item := range list.NonSkipped() {
tpl, err := meta.tf.Add(ctx, item.TFAddr())
if err != nil {
return fmt.Errorf("generating resource template for %s: %w", item.TFAddr(), err)
Expand All @@ -248,11 +263,8 @@ func (meta *Meta) Import(ctx context.Context, list ImportList) error {
defer os.Remove(cfgFile)

// Import resources
for idx, item := range list {
if item.Skip {
continue
}
fmt.Printf("[%d/%d] Importing %q as %s\n", idx+1, len(list), item.ResourceID, item.TFAddr())
for idx, item := range list.NonSkipped() {
fmt.Printf("[%d/%d] Importing %q as %s\n", idx+1, len(list.NonSkipped()), item.ResourceID, item.TFAddr())
if err := meta.tf.Import(ctx, item.TFAddr(), item.ResourceID); err != nil {
return err
}
Expand All @@ -274,7 +286,8 @@ func (cfg ConfigInfo) DumpHCL(w io.Writer) (int, error) {

func (meta *Meta) StateToConfig(ctx context.Context, list ImportList) (ConfigInfos, error) {
out := ConfigInfos{}
for _, item := range list {

for _, item := range list.NonSkipped() {
if item.Skip {
continue
}
Expand All @@ -301,39 +314,7 @@ func (meta *Meta) StateToConfig(ctx context.Context, list ImportList) (ConfigInf
}

func (meta *Meta) ResolveDependency(ctx context.Context, configs ConfigInfos) (ConfigInfos, error) {
client := meta.auth.NewResourceGroupClient()

exportOpt := "SkipAllParameterization"
future, err := client.ExportTemplate(ctx, meta.resourceGroup, resources.ExportTemplateRequest{
ResourcesProperty: &[]string{"*"},
Options: &exportOpt,
})
if err != nil {
return nil, fmt.Errorf("exporting arm template of resource group %s: %w", meta.resourceGroup, err)
}

if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return nil, fmt.Errorf("waiting for exporting arm template of resource group %s: %w", meta.resourceGroup, err)
}

result, err := future.Result(client)
if err != nil {
return nil, fmt.Errorf("getting the arm template of resource group %s: %w", meta.resourceGroup, err)
}

// The response has been read into the ".Template" field as an interface, and the reader has been drained.
// As we have defined some (useful) types for the arm template, so we will do a json marshal then unmarshal here
// to convert the ".Template" (interface{}) into that artificial type.
raw, err := json.Marshal(result.Template)
if err != nil {
return nil, fmt.Errorf("marshalling the template: %w", err)
}
var tpl armtemplate.Template
if err := json.Unmarshal(raw, &tpl); err != nil {
return nil, fmt.Errorf("unmarshalling the template: %w", err)
}

depInfo := tpl.DependencyInfo(meta.resourceGroup)
depInfo := meta.armTemplate.DependencyInfo()

configSet := map[armtemplate.ResourceId]ConfigInfo{}
for _, cfg := range configs {
Expand All @@ -353,7 +334,7 @@ func (meta *Meta) ResolveDependency(ctx context.Context, configs ConfigInfos) (C
}
// This should never happen
if _, ok := depInfo[armId]; !ok {
return nil, fmt.Errorf("resource %q appeared in the list result of resource group %q, but didn't show up in the arm template", armId.ID(meta.subscriptionId, meta.resourceGroup), meta.resourceGroup)
return nil, fmt.Errorf("can't find resource %q in the arm template", armId.ID(meta.subscriptionId, meta.resourceGroup))
}

meta.hclBlockAppendDependency(cfg.hcl.Body().Blocks()[0].Body(), depInfo[armId], configSet)
Expand Down
5 changes: 2 additions & 3 deletions internal/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,11 @@ func Run(ctx context.Context, rg string) error {
return err
}

ids, err := meta.ListResources(ctx)
if err != nil {
if err := meta.ExportArmTemplate(ctx); err != nil {
return err
}

importList, err := meta.ResolveImportList(ctx, ids)
importList, err := meta.ResolveImportList(ctx)
if err != nil {
return err
}
Expand Down

0 comments on commit 97e3b25

Please sign in to comment.