From 7c79722a30f669768f535a8983a1b406bdb4a372 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 2 May 2019 11:52:40 -0400 Subject: [PATCH] Initial cut for Create/Updating google groups New collaborators: 6103BF59 Davanum Srinivas D39F838B Christoph Blecker `go run reconcile.go` `go run reconcile.go --dry-run` (to test it) Sync's the existing google groups that i am aware of, please see: groups.yaml Change-Id: I33d0eccb3f9087d339323ca35a425dff2416b9d4 --- .git-crypt/.gitattributes | 4 + ...433CA4A347A8515465ED50B34A59A9D39F838B.gpg | Bin 0 -> 727 bytes ...7E5FD880EB089F2317796780D83A796103BF59.gpg | Bin 0 -> 727 bytes .gitattributes | 1 + groups/config.yaml | 10 + groups/go.mod | 10 + groups/go.sum | 60 +++ groups/groups.yaml | 86 ++++ .../k8s-infra-test-project-1896690daeb3.json | Bin 0 -> 2385 bytes groups/reconcile.go | 405 ++++++++++++++++++ 10 files changed, 576 insertions(+) create mode 100644 .git-crypt/.gitattributes create mode 100644 .git-crypt/keys/default/0/10433CA4A347A8515465ED50B34A59A9D39F838B.gpg create mode 100644 .git-crypt/keys/default/0/A67E5FD880EB089F2317796780D83A796103BF59.gpg create mode 100644 .gitattributes create mode 100644 groups/config.yaml create mode 100644 groups/go.mod create mode 100644 groups/go.sum create mode 100644 groups/groups.yaml create mode 100644 groups/k8s-infra-test-project-1896690daeb3.json create mode 100644 groups/reconcile.go diff --git a/.git-crypt/.gitattributes b/.git-crypt/.gitattributes new file mode 100644 index 00000000000..665b10e8f03 --- /dev/null +++ b/.git-crypt/.gitattributes @@ -0,0 +1,4 @@ +# Do not edit this file. To specify the files to encrypt, create your own +# .gitattributes file in the directory where your files are. +* !filter !diff +*.gpg binary diff --git a/.git-crypt/keys/default/0/10433CA4A347A8515465ED50B34A59A9D39F838B.gpg b/.git-crypt/keys/default/0/10433CA4A347A8515465ED50B34A59A9D39F838B.gpg new file mode 100644 index 0000000000000000000000000000000000000000..209ba216f690711a8d3c2066e16c3e7dd8f806f0 GIT binary patch literal 727 zcmV;|0x1230t^H2yzH7}N~-Mv5B(KQG}`t*S-4Ch3RSoOJI7y(S;JjaOUCG#3(l&~ z+L=y95fO|_wgduvX!XGyz+Iw(X>!{?a^mLrmMI>bL2mqByp)Y3hydh-4HPzpC3Fp- znzJ+{`xdKlnzQ=C1m=Ed^pDK8KOeBycL&KBsrMvaC-!NI%3s}oc_BN&x@~ z2`%xW?mMx z33a8jki-B&G}`{j`IqX*9x&HONyrQz!y3+44j|Plf*Yn9j|FPxF1wFDRf|jMr9B-W zKMbI|jw!sWv3I?dTVRMZQO)(Ca%KCtDUSoNbE9VCP6#eJk#V@a69Spac$pn?Qbwsf zc5s_w?h>Y)Mx;fqs88K|3JPp!QY~DsaF4#i5ep0}+m(cyNgMtn1;qboSgAsx_+TBH z>Zp+g2{2nb^z$Lx*KfL$K;5R55R$wxT9Cc(Yhrij+ca>|1pMU<&h6CoDH!(ud^_W= zC(|8?I$jFHwAW2qJ#5mz1p(6v{s8K8NLM_<=qzBqv2PU)^osr?Us*hdl}+mrr|A+O zK_$BRb|uH+5}cIjGZl!(=D{pY;A*NwU?(+l2K7)$SHs0A6;<4j4?ezyV{}AO#rKM) zr|r5y|JKTf%OzBop6m>V$`c}0=2YK57l&ygHOFFbtPpd$ad^vv1DQJ=>pei5b&jF z-kD6{2=#8A$w7O^Wc0r{-@JKW?x?GAXqr~WL@Do3QFrsH%~bCvW_~*5r4c)jl(p?>5NhaGDXiaGjR z?RmxE^}cwTY_yyrTCR!5ku?YasEN;TXak4{ZlS$b$6*aN5d*87Nb1beQyoLoh(tQrwy13+##=~l0}pl^V(oKG)}m8w@c?T#iuyFW2UQPr6YPDUV0e;gcC=6521I?YQ!XL zz(&lYv)-q9#DEoZADJqhMILsmjv_zzid41gV6Lk;*uG!P4L*zDe_OZ`+|e1d4dRgi z%6i2;NX(R& z*X#DWNqUk%43PuBTRxS4~#2>rB^;!qLKEyLF6 zOPX`)NJpgj|7z3o{^Riug3>75Yf!i3JP$O0L&6rIC=RrT_cnh%y3bG=l=ABrL%|R*15_kwXoCQz81)cZ^c&W^Yse z`0wwAH3l9lns}(rZfcEE8hW^TErSyGx<5S1`5f#(q;R}P&+#^Ca&$2JAmC_B%rS>L zqT(TKYL<48&4ml7@C5W=V@^44Yn*2WaJ78f?8t4knN41L;W6D+9Cz4V-tEqK&r{gC z5Z(8e>vfW77S?RYT8SXv=s`WQHQ{y|rKZINu5F%xYZYlb5U@{ga-#4muYDC+T}?|9 zrPjjQtj4!uuw&GpDU&AaL#vFYnwXVlRY5A;C6G<3Z?<*vn;f@j!x0i(DJcw%7I)dq z;~)VI01(~IF&s?7!0ODK2Zoi(^YCXrOt*Y%7S5u;cIjsmyj#tc4ml2Z zg_s|fMl%y+dP6C;Aq^3Bj3m7(lAm+-b^D*y*deka?)2b4f z&T%_qgE__A4x;e759L;M)bFczWbb?1@%4h_g`RW=O_Iab_dJC7(}QmdYK^v;suO2^ z(CfsH>841og`qly*bDutP~HygZ!<9b*EstFG*=8|y21%owb64>lSeds&zK&4_yBLGuWigs1GuEsX->xSAK`S|u$nc^`r(<#LW?8-K8&9d zTraac*v&HsbDg07u`fl;)lauNj9RqGeB`Y%cvUg*|7-7WIBoWd{LGtwhE|c)bcpEU z=ND&@u8G!yf$R~KITp$3eL{0?m<8gC#yV~!1Ge2FO|S5+8#IQ-N+0cyPt3M^uN8nb z<-sh&uu9&YdSu=Qp0nNrYI+cXQL+QB1HL@%lJ@5%rasx|6}O6qg}LmL_O57J<~FzV zIMmezVE$|8eYC@5jQz7z2f*sj+S%WkIR;}&d#t*uT}*o_mQ4c{HZ>}hdRtT<%=J== zTzjPTkWT>{*R=~;B6L%SqWUnt(Y&^`EHKHB^^^2?n@E-wi)OPBZZAs#b_d{UMSlqu zD@oSox&WJvn+V%}SK@5A?rIhgFiT$X(6!T``Xsa?RugT_e8aG*W*qfN`#OA?Kp0W? zrXJYvdTy zXF-{F`AI$)$UmAk(|P1}#V7=YE5kFsnyhuJxE3Gjdc#9So+GOMmMDCLAs>7-P0*YG z*jm>;ha|n~2EiYHwJ5$~O+9Wi|3xu0t&|J<>0ksEv0rpZH%&s73wk6YIqjEz)`)AK zJ8M2dk~kZ_03Z3_)ECaVQH#|HPC;Uq)spQFg-I)^qa-O-P^tF{zG^-I#RVN$LStW2 zxQH!S-2Xt8R~38$3#f7lKm8V5>-Ue+^hgNjx-@gZFvDMG+rJbcG5Oi9qw27CyBsn4 zH4=Ex(Jw(SDb(LLN>@}4u(BrE#m2~8E(jh-QqOC?R*eO73zGUUK_^n;CXfg{D3);y zjv?e5OLNMCV*&;jw4zdtvOKAAyIm`rMyfaU?#s?|N?(WmYAB*-c4d@FdZ=|+6xg45 z$@zK0$YQ$KUl5mj`x$taH2<~^MUAWN%mbjmO9EnnPEGYrljxd*RdQ*p*S};#_A$Ma z^j>e3*FM>7sLew%YE>X&TJ0=2ykDuu-NZ#QfWU2Qc7~sBC4XlRs!A4o0<;-CwuP1? z`{gOfNbgg9-t*`q&kNzFm;M#ll)tD|>7W&Ltcl7Y!MVK+IqrQIChca+Gt7w{9MTn% z4!&UzIkK(D{mr0kq!N;0wE1}-&SbJVPxUx{RxPsGmrAsR(gndN45GhP!bn$gIDU*r zbA_W_+f%0ZYMftAj0zq`DwH0GMWF?aYJG?4(dxvSjXUcU+A9HG?l)<{M?=ZcFWXoR zVa<58)To?gAcb5=sx;RJBr&#bWB@fd4j76TW^uYQudPzoq5Cz|B7iK^tE%@gk<@E4 z!E>`b?$`%Ub*^qcpxP*)S)0qwBu=|;bH0Ae^J}m%4=!{}7vAM?W8*a!*4<4ILk>%p znZ09=f^m8SxW?XM$V5qofSLgQffrmk80$QW<2AXdF)9$8r9!Ccvx#h{;U_kjzr^Mp z;X{2!ly@m$2YukNii(C$(g#f zixrnS$reI2C`K4%_0KW8Xw?1L`RpI2H@O=&N&A>A>mLBQcsgVb+vXSFTRCl|_(y{nN$1!xHh>#Vv!fXH82)Lx&|34r^@==w3HiI! z-{fGon@f0p**-F7(A2JhK0K)me3tr>X`*D3A2 zVaoaW8Lb}oqJg%v9SxUvCraeRu7BW&sI8=Pjn-qTPlyY)@gnOMg^f;HGj-)UYhGpx zS82o_n$&PJ7Ee(R?U1et&64!NHXc-%(-i{u@3zMgDBK}mZDY7K>`0j8YhiGN;*JF2 z&n2lrgL}ht4XoI3*-_0ZZJe{;Jl@DMIb=&y6&BG8 zoN=pm9su(@)wL~?h9P#1=C1(46;1sPOqI] [-dry-run] +Command line flags override config values. +`, os.Args[0]) + flag.PrintDefaults() +} + +var config Config +var groupsConfig GroupsConfig + +func main() { + configFilePath := flag.String("config", "config.yaml", "the config file in yaml format") + dryRun := flag.Bool("dry-run", false, "do not push anything to google groups") + + flag.Usage = Usage + flag.Parse() + + err := readConfig(configFilePath, dryRun) + if err != nil { + log.Fatal(err) + } + + err = readGroupsConfig(config.GroupsFile) + if err != nil { + log.Fatal(err) + } + + jsonCredentials, err := ioutil.ReadFile(config.TokenFile) + if err != nil { + log.Fatal(err) + } + + credential, err := google.JWTConfigFromJSON(jsonCredentials, admin.AdminDirectoryUserReadonlyScope, + admin.AdminDirectoryGroupScope, + admin.AdminDirectoryGroupMemberScope, + groupssettings.AppsGroupsSettingsScope) + if err != nil { + log.Fatalf(fmt.Sprintf("Unable to parse client secret file to config: %v\n. "+ + "Please run 'git-crypt unlock'", err)) + } + credential.Subject = config.BotID + + client := credential.Client(context.Background()) + srv, err := admin.New(client) + if err != nil { + log.Fatalf("Unable to retrieve directory Client %v", err) + } + + srv2, err := groupssettings.New(client) + if err != nil { + log.Fatalf("Unable to retrieve groupssettings Service %v", err) + } + + fmt.Println(" =================== Current Status ======================") + err = printGroupMembersAndSettings(srv, srv2) + if err != nil { + log.Fatal(err) + } + + fmt.Println(" ======================= Updates =========================") + for _, g := range groupsConfig.Groups { + if !strings.HasPrefix(g.EmailId, "k8s-infra-") { + log.Fatalf("We can reconcile only groups that start with 'k8s-infra-' prefix") + } + err = createGroupIfNecessary(srv, g.EmailId, g.Description) + if err != nil { + log.Fatal(err) + } + err = updateGroupSettingsToAllowExternalMembers(srv2, g.EmailId) + if err != nil { + log.Fatal(err) + } + err = addMembersToGroup(srv, g.EmailId, g.Members) + if err != nil { + log.Fatal(err) + } + err = removeMembersFromGroup(srv, g.EmailId, g.Members) + if err != nil { + log.Fatal(err) + } + } + err = deleteGroupsIfNecessary(srv) + if err != nil { + log.Fatal(err) + } +} + +func readConfig(configFilePath *string, dryRun *bool) error { + content, err := ioutil.ReadFile(*configFilePath) + if err != nil { + return fmt.Errorf("error reading config from file: %v", err) + } + if err = yaml.Unmarshal(content, &config); err != nil { + return fmt.Errorf("error reading config: %v", err) + } + if dryRun != nil { + config.DryRun = *dryRun + } + return err +} + +func readGroupsConfig(groupsConfigFilePath string) error { + var content []byte + var err error + groupsUrl, err := url.ParseRequestURI(groupsConfigFilePath) + if err == nil { + // We have a URL, so try reading from it + if len(groupsUrl.Host) > 0 { + if content, err = readFromUrl(groupsUrl); err != nil { + return fmt.Errorf("error reading groups config from file: %v", err) + } + } + } else { + // We don't have a URL, we have a file path, so try reading from the file + if content, err = ioutil.ReadFile(groupsConfigFilePath); err != nil { + return fmt.Errorf("error reading groups config from file: %v", err) + } + } + if err = yaml.Unmarshal(content, &groupsConfig); err != nil { + return fmt.Errorf("error reading groups config: %v", err) + } + return nil +} + +// readFromUrl reads the rule file from provided URL. +func readFromUrl(u *url.URL) ([]byte, error) { + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + // timeout the request after 30 seconds + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return ioutil.ReadAll(resp.Body) +} + +func printGroupMembersAndSettings(srv *admin.Service, srv2 *groupssettings.Service) error { + g, err := srv.Groups.List().Customer("my_customer").OrderBy("email").Do() + if err != nil { + return fmt.Errorf("unable to retrieve users in domain: %v", err) + } + if len(g.Groups) == 0 { + fmt.Print("No groups found.\n") + return nil + } + for _, g := range g.Groups { + // Don't touch existing mailing lists, we should + // always prefix with "k8s-infra-" + if !strings.HasPrefix(g.Email, "k8s-infra-") { + continue + } + log.Printf("%s\n", g.Email) + + g2, err := srv2.Groups.Get(g.Email).Do() + if err != nil { + return fmt.Errorf("unable to retrieve group info for group %s: %v", g.Email, err) + } + log.Printf(">> Allow external members %s\n", g2.AllowExternalMembers) + + l, err := srv.Members.List(g.Email).Do() + if err != nil { + return fmt.Errorf("unable to retrieve members in group : %v", err) + } + + if len(l.Members) == 0 { + fmt.Print("No members found in group.\n") + } else { + for _, m := range l.Members { + log.Printf(">>> %s (%s)\n", m.Email, m.Role) + } + } + log.Printf("\n") + + } + return nil +} + +func createGroupIfNecessary(srv *admin.Service, groupEmailId string, description string) error { + _, err := srv.Groups.Get(groupEmailId).Do() + if err != nil { + if apierr, ok := err.(*googleapi.Error); ok && apierr.Code == http.StatusNotFound { + if config.DryRun { + log.Printf("dry-run : skipping creation of group %s\n", groupEmailId) + } else { + log.Printf("Trying to create group: %s\n", groupEmailId) + g4, err := srv.Groups.Insert(&admin.Group{ + Email: groupEmailId, + Name: description, + Description: "Kubernetes wg-k8s-infra test group #2", + }).Do() + if err != nil { + return fmt.Errorf("unable to add new group %s: %v", groupEmailId, err) + } else { + log.Printf("> Successfully created group %s\n", g4.Email) + } + } + } + return fmt.Errorf("unable to fetch group: %#v", err.Error()) + } + return nil +} + +func deleteGroupsIfNecessary(service *admin.Service) error { + g, err := service.Groups.List().Customer("my_customer").OrderBy("email").Do() + if err != nil { + return fmt.Errorf("unable to retrieve users in domain: %v", err) + } + if len(g.Groups) == 0 { + fmt.Print("No groups found.\n") + return nil + } + for _, g := range g.Groups { + // Don't touch existing mailing lists, we should + // always prefix with "k8s-infra-" + if !strings.HasPrefix(g.Email, "k8s-infra-") { + continue + } + found := false + for _, g2 := range groupsConfig.Groups { + if g2.EmailId == g.Email { + found = true + break + } + } + if found { + continue + } + // We did not find the group in our groups.xml, so delete the group + if config.DryRun { + log.Printf("dry-run : Skipping removing group %s\n", g.Email) + } else { + err := service.Groups.Delete(g.Email).Do() + if err != nil { + return fmt.Errorf("unable to remove group %s : %v", g.Email, err) + } else { + log.Printf("Removing group %s\n", g.Email) + } + } + + } + return nil +} + +func updateGroupSettingsToAllowExternalMembers(srv *groupssettings.Service, groupEmailId string) error { + g2, err := srv.Groups.Get(groupEmailId).Do() + if err != nil { + if apierr, ok := err.(*googleapi.Error); ok && apierr.Code == http.StatusNotFound { + log.Printf("skipping updating group settings as group %s has not yet been created\n", groupEmailId) + return nil + } + return fmt.Errorf("unable to retrieve group info for group %s: %v", groupEmailId, err) + } + if g2.AllowExternalMembers != "true" { + if config.DryRun { + log.Printf("dry-run : skipping updating group settings\n") + } else { + _, err := srv.Groups.Patch(groupEmailId, &groupssettings.Groups{ + AllowExternalMembers: "true", + }).Do() + if err != nil { + return fmt.Errorf("unable to update group info for group %s: %v", groupEmailId, err) + } else { + log.Printf("> Successfully updated group settings for %s to allow external members\n", groupEmailId) + } + } + } + return nil +} + +func addMembersToGroup(service *admin.Service, groupEmailId string, members []string) error { + l, err := service.Members.List(groupEmailId).Do() + if err != nil { + if apierr, ok := err.(*googleapi.Error); ok && apierr.Code == http.StatusNotFound { + log.Printf("skipping adding members to group %s as it has not yet been created\n", groupEmailId) + return nil + } + return fmt.Errorf("unable to retrieve members in group %s: %v", groupEmailId, err) + } + + for _, m := range members { + found := false + if len(l.Members) > 0 { + for _, m2 := range l.Members { + if m2.Email == m { + found = true + break + } + } + } + if found { + continue + } + // We did not find the person in the google group, so we add them + if config.DryRun { + log.Printf("dry-run : Skipping adding %s to %s\n", m, groupEmailId) + } else { + _, err := service.Members.Insert(groupEmailId, &admin.Member{ + Email: m, + Role: "MEMBER", + }).Do() + if err != nil { + return fmt.Errorf("unable to add %s to %s : %v", m, groupEmailId, err) + } else { + log.Printf("Added %s to %s as a MEMBER\n", m, groupEmailId) + } + } + } + return nil +} + +func removeMembersFromGroup(service *admin.Service, groupEmailId string, members []string) error { + l, err := service.Members.List(groupEmailId).Do() + if err != nil { + if apierr, ok := err.(*googleapi.Error); ok && apierr.Code == http.StatusNotFound { + log.Printf("skipping removing members group %s as group has not yet been created\n", groupEmailId) + return nil + } + return fmt.Errorf("unable to retrieve members in group %s: %v", groupEmailId, err) + } + + for _, m := range l.Members { + found := false + if len(members) > 0 { + for _, m2 := range members { + if m2 == m.Email { + found = true + break + } + } + } + if found { + continue + } + // a person was deleted from a group, let's remove them + if config.DryRun { + log.Printf("dry-run : Skipping removing %s from %s\n", m.Email, groupEmailId) + } else { + err := service.Members.Delete(groupEmailId, m.Email).Do() + if err != nil { + return fmt.Errorf("unable to remove %s from %s : %v", m.Email, groupEmailId, err) + } else { + log.Printf("Removing %s from %s as a MEMBER\n", m.Email, groupEmailId) + } + } + } + return nil +}