Skip to content

Commit

Permalink
add cookiecloud sync cookies / import sites features
Browse files Browse the repository at this point in the history
  • Loading branch information
sagan committed Dec 5, 2023
1 parent f9563cc commit 5e1cb46
Show file tree
Hide file tree
Showing 11 changed files with 617 additions and 33 deletions.
2 changes: 2 additions & 0 deletions cmd/cookiecloud/all/all.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,7 @@ package all

import (
_ "github.com/sagan/ptool/cmd/cookiecloud"
_ "github.com/sagan/ptool/cmd/cookiecloud/importsites"
_ "github.com/sagan/ptool/cmd/cookiecloud/status"
_ "github.com/sagan/ptool/cmd/cookiecloud/sync"
)
111 changes: 109 additions & 2 deletions cmd/cookiecloud/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,29 @@ package cookiecloud
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"sort"
"strings"

"github.com/sagan/ptool/util"
"github.com/sagan/ptool/util/crypto"
)

type Ccdata_struct struct {
Domain string
Uuid string
Sites []string
Data *CookiecloudData
}

type Cookie struct {
Domain string
Name string
Value string
Path string
}

type CookieCloudBody struct {
Uuid string `json:"uuid,omitempty"`
Encrypted string `json:"encrypted,omitempty"`
Expand All @@ -19,15 +36,27 @@ type CookiecloudData struct {
Cookie_data map[string][]map[string]any `json:"cookie_data"`
}

func GetCookiecloudData(server string, uuid string, password string) (*CookiecloudData, error) {
func GetCookiecloudData(server string, uuid string, password string, proxy string) (*CookiecloudData, error) {
if server == "" || uuid == "" || password == "" {
return nil, fmt.Errorf("all params of server,uuid,password must be provided")
}
if !strings.HasSuffix(server, "/") {
server += "/"
}
var httpClient *http.Client
if proxy != "" {
proxyUrl, err := url.Parse(proxy)
if err != nil {
return nil, fmt.Errorf("failed to parse proxy %s: %v", proxy, err)
}
httpClient = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyUrl),
},
}
}
var data *CookieCloudBody
err := util.FetchJson(server+"get/"+uuid, &data, nil, "", "", nil)
err := util.FetchJson(server+"get/"+uuid, &data, httpClient, "", "", nil)
if err != nil || data == nil {
return nil, fmt.Errorf("failed to get cookiecloud data: err=%v, null data=%t", err, data == nil)
}
Expand All @@ -43,3 +72,81 @@ func GetCookiecloudData(server string, uuid string, password string) (*Cookieclo
}
return cookiecloudData, nil
}

func (cookiecloudData *CookiecloudData) GetEffectiveCookie(urlOrHost string) (string, error) {
hostname := ""
path := "/"
if util.IsUrl(urlOrHost) {
urlObj, err := url.Parse(urlOrHost)
if err != nil {
return "", fmt.Errorf("arg is not a valid url: %v", err)
}
hostname = urlObj.Hostname()
path = urlObj.Path
}
if hostname == "" {
return "", fmt.Errorf("hostname can not be empty")
}
effectiveCookies := []*Cookie{}
keys := []string{hostname, "." + hostname}
for _, key := range keys {
cookies, ok := cookiecloudData.Cookie_data[key]
if !ok {
continue
}
for _, cookie := range cookies {
if cookie == nil {
continue
}
cookieDomain, _ := cookie["domain"].(string)
if cookieDomain != hostname && cookieDomain != "."+hostname {
continue
}
cookiePath, _ := cookie["path"].(string)
if cookiePath == "" {
cookiePath = "/"
}
if !strings.HasPrefix(path, cookiePath) {
continue
}
// cookiecloud 导出的 cookies 里的 expirationDate 为 float 类型。意义不明确,暂不使用。
cookieName, _ := cookie["name"].(string)
cookieValue, _ := cookie["value"].(string)
// RFC 似乎允许 empty cookie ?
if cookieName == "" || cookieValue == "" {
continue
}
effectiveCookies = append(effectiveCookies, &Cookie{
Domain: cookieDomain,
Path: cookiePath,
Name: cookieName,
Value: cookieValue,
})
}
}
if len(effectiveCookies) == 0 {
return "", nil
}
sort.SliceStable(effectiveCookies, func(i, j int) bool {
a := effectiveCookies[i]
b := effectiveCookies[j]
if a.Domain != b.Domain {
return false
}
// longest path first
if len(a.Path) != len(b.Path) {
return len(a.Path) > len(b.Path)
}
return false
})
effectiveCookies = util.UniqueSliceFn(effectiveCookies, func(cookie *Cookie) string {
return cookie.Name
})
cookieStr := ""
sep := ""
for _, cookie := range effectiveCookies {
cookieStr += sep + cookie.Name + "=" + cookie.Value
sep = "; "
}
return cookieStr, nil
}
37 changes: 35 additions & 2 deletions cmd/cookiecloud/cookiecloud.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,27 @@
package cookiecloud

import (
"strings"

"github.com/spf13/cobra"

"github.com/sagan/ptool/cmd"
"github.com/sagan/ptool/config"
)

var Command = &cobra.Command{
Use: "cookiecloud",
Short: "Use cookiecloud to sync cookies or import sites.",
Long: `Use cookiecloud to sync cookies or import sites.
Short: "Use cookiecloud to sync site cookies or import sites.",
Long: `Use cookiecloud to sync site cookies or import sites.
To use this feature, add the cookiecloud servers to config file, e.g. :
ptool.toml
----------
[[cookieclouds]]
server = 'https://cookiecloud.example.com'
uuid = 'uuid'
password = 'password'
----------
See also:
* CookieCloud: https://github.com/easychen/CookieCloud`,
Expand All @@ -19,3 +31,24 @@ See also:
func init() {
cmd.RootCmd.AddCommand(Command)
}

func ParseProfile(profile string) []*config.CookiecloudConfigStruct {
cookiecloudProfiles := []*config.CookiecloudConfigStruct{}
if profile == "" {
for _, profile := range config.Get().Cookieclouds {
if profile.Disabled {
continue
}
cookiecloudProfiles = append(cookiecloudProfiles, profile)
}
} else {
names := strings.Split(profile, ",")
for _, name := range names {
profile := config.GetCookiecloudConfig(name)
if profile != nil {
cookiecloudProfiles = append(cookiecloudProfiles, profile)
}
}
}
return cookiecloudProfiles
}
159 changes: 159 additions & 0 deletions cmd/cookiecloud/importsites/importsites.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package importsites

import (
"fmt"
"slices"
"strings"

log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"github.com/sagan/ptool/cmd/cookiecloud"
"github.com/sagan/ptool/config"
"github.com/sagan/ptool/site"
"github.com/sagan/ptool/site/tpl"
"github.com/sagan/ptool/util"
)

var (
doAction = false
profile = ""
)

var command = &cobra.Command{
Use: "import",
Annotations: map[string]string{"cobra-prompt-dynamic-suggestions": "cookiecloud.import"},
Short: "Import sites from cookies of cookiecloud servers.",
Long: `Import sites from cookies of cookiecloud servers.
It will get latest cookies from cookiecloud servers, find sites that do NOT exist in config file currently,
Test their cookies are valid, then add them to config file.
It will ask for confirm before updating config file, unless --do flag is set.
Be aware that all existing comments in config file will be LOST when updating config file.`,
RunE: importsites,
}

func init() {
command.Flags().BoolVarP(&doAction, "do", "", false, "Do update the config file without confirm. Be aware that all existing comments in config file will be LOST")
command.Flags().StringVarP(&profile, "profile", "", "", "Comma-separated string, Set the used cookiecloud profile name(s). If not set, All cookiecloud profiles in config will be used")
cookiecloud.Command.AddCommand(command)
}

func importsites(cmd *cobra.Command, args []string) error {
cntError := int64(0)
cookiecloudProfiles := cookiecloud.ParseProfile(profile)
if len(cookiecloudProfiles) == 0 {
return fmt.Errorf("no cookiecloud profile specified or found")
}
cookiecloudDatas := []cookiecloud.Ccdata_struct{}
for _, profile := range cookiecloudProfiles {
data, err := cookiecloud.GetCookiecloudData(profile.Server, profile.Uuid, profile.Password, profile.Proxy)
if err != nil {
log.Errorf("Cookiecloud server %s (uuid %s) connection failed: %v\n", profile.Server, profile.Uuid, err)
cntError++
} else {
log.Infof("Cookiecloud server %s (uuid %s) connection ok: %d site cookies found\n",
profile.Server, profile.Uuid, len(data.Cookie_data))
cookiecloudDatas = append(cookiecloudDatas, cookiecloud.Ccdata_struct{
Domain: util.GetUrlDomain(profile.Server),
Uuid: profile.Uuid,
Sites: profile.Sites,
Data: data,
})
}
}
if len(cookiecloudDatas) == 0 {
return fmt.Errorf("no cookiecloud server can be connected")
}

addSites := []*config.SiteConfigStruct{}
tplExistingFlags := map[string]bool{}
for _, tplname := range tpl.SITENAMES {
tplInfo := tpl.SITES[tplname]
for _, site := range config.Get().Sites {
if site.Type == tplname || slices.Index(tplInfo.Aliases, site.Type) != -1 {
tplExistingFlags[tplname] = true
break
}
}
if sitename, _ := tpl.GuessSiteByDomain(util.ParseUrlHostname(tplInfo.Url), ""); sitename != "" {
tplExistingFlags[tplname] = true
}
}
for _, cookiecloudData := range cookiecloudDatas {
for _, tplname := range tpl.SITENAMES {
if tplExistingFlags[tplname] {
continue
}
cookie, _ := cookiecloudData.Data.GetEffectiveCookie(tpl.SITES[tplname].Url)
if cookie == "" {
continue
}
newsiteconfig := &config.SiteConfigStruct{Type: tplname, Cookie: cookie}
siteInstance, err := site.CreateSiteInternal(tplname, newsiteconfig, config.Get())
if err != nil {
log.Debugf("New Site %s from cookiecloud %s - %s is invalid (create instance error: %v",
tplname, cookiecloudData.Domain, cookiecloudData.Uuid, err)
continue
}
sitestatus, err := siteInstance.GetStatus()
if err != nil {
log.Debugf("New Site %s from cookiecloud %s - %s is invalid (get status error: %v",
tplname, cookiecloudData.Domain, cookiecloudData.Uuid, err)
continue
}
log.Infof("✓✓New site %s from cookiecloud %s - %s is valid (username: %s)",
tplname, cookiecloudData.Domain, cookiecloudData.Uuid, sitestatus.UserName)
sitename := ""
if config.GetSiteConfig(tplname) != nil {
i := 1
for {
sitename = fmt.Sprint(tplname, i)
if config.GetSiteConfig(sitename) == nil {
break
}
i++
}
}
log.Infof("Add new site type=%s, name=%s", tplname, sitename)
addSites = append(addSites, &config.SiteConfigStruct{
Name: sitename,
Type: tplname,
Cookie: cookie,
})
tplExistingFlags[tplname] = true
}
}

if len(addSites) > 0 {
fmt.Printf("✓new sites found (%d): %s", len(addSites),
strings.Join(util.Map(addSites, func(site *config.SiteConfigStruct) string {
return site.Type
}), ", "))

configFile := fmt.Sprintf("%s/%s", config.ConfigDir, config.ConfigFile)
fmt.Printf("\n")
if !doAction {
fmt.Printf("Will update the config file (%s). Please be aware that all existing comments will be LOST., are you sure? (yes/no): ", configFile)
input := ""
fmt.Scanf("%s", &input)
if input != "yes" {
return fmt.Errorf("abort")
}
}
config.UpdateSites(addSites)
err := config.Set()
if err == nil {
fmt.Printf("Successfully update config file %s", configFile)
} else {
log.Fatalf("Failed to update config file %s : %v", configFile, err)
}
} else {
fmt.Printf("!No new sites found in cookiecloud datas")
}

if cntError > 0 {
return fmt.Errorf("%d errors", cntError)
}
return nil
}

0 comments on commit 5e1cb46

Please sign in to comment.