Skip to content

Commit

Permalink
FII dividends report
Browse files Browse the repository at this point in the history
  • Loading branch information
@dude333
dude333 committed Apr 21, 2021
1 parent 07d7935 commit 015eec5
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 74 deletions.
15 changes: 14 additions & 1 deletion cli/cmd/fii.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,24 @@ Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
// Fund code
code, err := cmd.Flags().GetString("dividend")
if err != nil {
log.Println(err)
return
}
if err := rapina.FIIDividends(code); err != nil {
if code == "" {
_ = cmd.Help()
return
}

// Number of reports
n, err := cmd.Flags().GetInt("number")
if err != nil || n <= 0 {
n = 1
}

if err := rapina.FIIDividends(code, n); err != nil {
log.Println(err)
}
},
Expand All @@ -53,6 +65,7 @@ to quickly create a Cobra application.`,
func init() {
rootCmd.AddCommand(fiiCmd)
fiiCmd.Flags().StringP("dividend", "d", "", "Dividends for FII CODE")
fiiCmd.Flags().IntP("number", "n", 1, "number of reports")

// Here you will define your flags and configuration settings.

Expand Down
32 changes: 25 additions & 7 deletions fii.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,50 @@ import (
"github.com/dude333/rapina/parsers"
)

func FIIDividends(code string) error {
//
// FIIDividends prints the dividends from 'code' fund for 'n' months,
// starting from latest.
//
func FIIDividends(code string, n int) error {
db, err := openDatabase()
if err != nil {
return err
}
u := "https://sistemaswebb3-listados.b3.com.br"
code = strings.ToUpper(code)

fii, _ := parsers.NewFII(db, u)
fii, _ := parsers.NewFII(db)

cnpj, err := cnpj(fii, code)
if err != nil {
return err
}

err = fii.FetchFIIDividends(cnpj, n)

return err
}

//
// cnpj returns the CNPJ from FII code. It first checks the DB and, if not
// found, fetches from B3.
//
func cnpj(fii *parsers.FII, code string) (string, error) {
fiiDetails, err := fii.SelectFIIDetails(code)
if err != nil && err != sql.ErrNoRows {
fmt.Println("[x] error", err)
}
if err == nil && fiiDetails.DetailFund.CNPJ != "" {
fmt.Println("DB", code, fiiDetails.DetailFund.CNPJ)
return nil
return fiiDetails.DetailFund.CNPJ, nil
}

//
// Fetch online if DB fails
fiiDetails, err = fii.FetchFIIDetails(code)
if err != nil {
return err
return "", err
}

fmt.Println("online", code, fiiDetails.DetailFund.CNPJ)

return nil
return fiiDetails.DetailFund.CNPJ, nil
}
145 changes: 79 additions & 66 deletions parsers/fii.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,15 @@ import (
"github.com/pkg/errors"
)

/*
(1) list []string <= FetchFIIList
(2) loop list: fii *FII <= FetchFIIDetails
(3) db: insert code, cnpj
*/

// FII holds the infrastructure data.
type FII struct {
db *sql.DB
baseURL string
db *sql.DB
}

func NewFII(db *sql.DB, baseURL string) (*FII, error) {
// NewFII creates a new instace of FII.
func NewFII(db *sql.DB) (*FII, error) {
fii := &FII{
db: db,
baseURL: baseURL,
db: db, // will accept null db when caching is no needed
}
return fii, nil
}
Expand Down Expand Up @@ -87,7 +81,7 @@ func FetchFIIList(baseURL string) ([]string, error) {
return codes, nil
}

// FIIDetails details (main field: DetailFund.CNPJ)
// FIIDetails details (ID field: DetailFund.CNPJ)
type FIIDetails struct {
DetailFund struct {
Acronym string `json:"acronym"`
Expand Down Expand Up @@ -136,7 +130,10 @@ func (fii FII) FetchFIIDetails(fiiCode string) (*FIIDetails, error) {

data := fmt.Sprintf(`{"typeFund":7,"cnpj":"0","identifierFund":"%s"}`, fiiCode[0:4])
enc := base64.URLEncoding.EncodeToString([]byte(data))
fundDetailURL := JoinURL(fii.baseURL, `/fundsProxy/fundsCall/GetDetailFundSIG/`, enc)
fundDetailURL := JoinURL(
`https://sistemaswebb3-listados.b3.com.br/fundsProxy/fundsCall/GetDetailFundSIG/`,
enc,
)

tr := &http.Transport{
DisableCompression: true,
Expand Down Expand Up @@ -172,29 +169,17 @@ func (fii FII) FetchFIIDetails(fiiCode string) (*FIIDetails, error) {
return &fiiDetails, err
}

// JoinURL joins strings as URL paths
func JoinURL(base string, paths ...string) string {
p := path.Join(paths...)
return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/"))
}

func trimFIIDetails(f *FIIDetails) {
f.DetailFund.CNPJ = strings.TrimSpace(f.DetailFund.CNPJ)
f.DetailFund.Acronym = strings.TrimSpace(f.DetailFund.Acronym)
f.DetailFund.TradingCode = strings.TrimSpace(f.DetailFund.TradingCode)
}

/* ------------------------------------------ */

type id int
type data struct {
type Report struct {
Data []docID `json:"data"`
}
type docID struct {
ID id `json:"id"`
Descr string `json:"descricaoFundo"`
TipoDoc string `json:"tipoDocumento"`
Sit string `json:"situacaoDocumento"`
ID id `json:"id"`
Description string `json:"descricaoFundo"`
DocType string `json:"tipoDocumento"`
Status string `json:"situacaoDocumento"`
}

/*
Expand All @@ -217,32 +202,19 @@ type fiiYeld struct {
*/

//
// FetchFII gets the report IDs for one company and then the
// yeld montlhy reports.
// FetchFIIDividends gets the report IDs for one company ('cnpj') and then the
// yeld montlhy report for 'n' months, starting from the latest released.
//
func FetchFII(baseURL string) error {
func (fii FII) FetchFIIDividends(cnpj string, n int) error {
var ids []id
yeld := make(map[string]string, n)
if n <= 0 {
n = 1
}

c := colly.NewCollector()
yeld := make(map[string]string, 20)

// Handles the html report
c.OnHTML("tr", func(e *colly.HTMLElement) {
var fieldName string

e.ForEach("td", func(_ int, el *colly.HTMLElement) {
v := strings.Trim(el.Text, " \r\n")
if v != "" {
if fieldName == "" {
fieldName = v
} else {
// fmt.Printf("%s => %s\n", fieldName, v)
yeld[fieldName] = v
fieldName = ""
}
}
})

c.WithTransport(&http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
})

c.OnRequest(func(r *colly.Request) {
Expand All @@ -258,47 +230,88 @@ func FetchFII(baseURL string) error {
if !strings.Contains(r.Headers.Get("content-type"), "application/json") {
return
}
var d data
err := json.Unmarshal(r.Body, &d)
var report Report
err := json.Unmarshal(r.Body, &report)
if err != nil {
fmt.Println("json error:", err)
} else {
for _, x := range d.Data {
if x.Sit == "A" {
ids = append(ids, x.ID)
}
return
}
for _, d := range report.Data {
if d.Status == "A" {
ids = append(ids, d.ID)
}
}
})

// Parameters to list the report IDs for the last 'n' dividend reports
timestamp := strconv.FormatInt(int64(time.Now().UnixNano()/1e6), 10)
v := url.Values{
"d": []string{"2"},
"s": []string{"0"},
"l": []string{"2"}, // months
"l": []string{strconv.Itoa(n)}, // months
"o[0][dataEntrega]": []string{"desc"},
"tipoFundo": []string{"1"},
"cnpjFundo": []string{"14410722000129"},
"cnpjFundo": []string{cnpj},
"idCategoriaDocumento": []string{"14"},
"idTipoDocumento": []string{"41"},
"idEspecieDocumento": []string{"0"},
"situacao": []string{"A"},
"_": []string{"1609254186709"},
"_": []string{timestamp},
}

// Get the 'report IDs' for a given company (CNPJ)
u := JoinURL(baseURL, "/pesquisarGerenciadorDocumentosDados?", v.Encode())
// Get the 'report IDs' for a given company (CNPJ) -- returns JSON
u := "https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosDados" +
"?" + v.Encode()
if err := c.Visit(u); err != nil {
return err
}

// Get the yeld monthly report given the list of 'report IDs'
// Handles the html report
c.OnHTML("tr", func(e *colly.HTMLElement) {
var fieldName string
e.ForEach("td", func(_ int, el *colly.HTMLElement) {
v := strings.Trim(el.Text, " \r\n")
if v != "" {
if fieldName == "" {
fieldName = v
} else {
fmt.Printf("%-30s => %s\n", fieldName, v)
yeld[fieldName] = v
fieldName = ""
}
}
})
})

// Get the yeld monthly report given the list of 'report IDs' -- returns HTML
for _, i := range ids {
u = JoinURL(baseURL, fmt.Sprintf("/exibirDocumento?id=%d&cvm=true", i))
u = fmt.Sprintf("https://fnet.bmfbovespa.com.br/fnet/publico/exibirDocumento?id=%d&cvm=true", i)
if err := c.Visit(u); err != nil {
return err
}
fmt.Println("----------------------------")

// fmt.Printf("%+v\n", yeld)
}

return nil
}

/* ------- Utils ------- */

func IsUrl(str string) bool {
u, err := url.Parse(str)
return err == nil && u.Scheme != "" && u.Host != ""
}

// JoinURL joins strings as URL paths
func JoinURL(base string, paths ...string) string {
p := path.Join(paths...)
return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/"))
}

func trimFIIDetails(f *FIIDetails) {
f.DetailFund.CNPJ = strings.TrimSpace(f.DetailFund.CNPJ)
f.DetailFund.Acronym = strings.TrimSpace(f.DetailFund.Acronym)
f.DetailFund.TradingCode = strings.TrimSpace(f.DetailFund.TradingCode)
}

0 comments on commit 015eec5

Please sign in to comment.