Permalink
Browse files

Update logic import & export function

  • Loading branch information...
dtduc91 committed Nov 16, 2017
1 parent 16a5810 commit 2dd0f8446dc8ba495f5096a8ebcced730debaf61
Showing with 83 additions and 49 deletions.
  1. +17 −1 bulkRequest.go
  2. +10 −10 bulkRequest_test.go
  3. +12 −14 import.go
  4. +1 −14 importWithBulkRequest.go
  5. +43 −10 main.go
View
@@ -19,6 +19,17 @@ import (
"github.com/kintone/go-kintone"
)
const (
// ConstBulkRequestLimitRecordOption set option: record per bulkRequest
ConstBulkRequestLimitRecordOption = 100
// ConstBulkRequestLimitRequest kintone limited: The request count per bulkRequest
ConstBulkRequestLimitRequest = 20
// ConstRecordsLimitPerRequest kintone limited: The records count per request
ConstRecordsLimitPerRequest = 100
)
// BulkRequestItem item in the bulkRequest array
type BulkRequestItem struct {
Method string `json:"method"`
@@ -48,20 +59,24 @@ type BulkRequestsErrors struct {
Results []*BulkRequestsError `json:"results"`
}
// DataResponseBulkPOST structure
type DataResponseBulkPOST struct {
Results []interface{} `json:"results,string"`
}
// DataRequestRecordsPOST structure
type DataRequestRecordsPOST struct {
App uint64 `json:"app,string"`
Records []*kintone.Record `json:"records"`
}
//DataRequestRecordPUT structure
type DataRequestRecordPUT struct {
ID uint64 `json:"id,string"`
Record *kintone.Record `json:"record,string"`
}
// DataRequestRecordPUTByKey structure
type DataRequestRecordPUTByKey struct {
UpdateKey *kintone.UpdateKey `json:"updateKey,string"`
Record *kintone.Record `json:"record,string"`
@@ -81,7 +96,7 @@ func (recordsPut *DataRequestRecordsPUT) SetRecord(record *kintone.Record) {
}
// SetRecord set data record for PUT method
// SetRecordWithKey set data record for PUT method
func (recordsPut *DataRequestRecordsPUT) SetRecordWithKey(record *kintone.Record, keyCode string) {
updateKey := &kintone.UpdateKey{FieldCode: keyCode, Field: record.Fields[keyCode].(kintone.UpdateKeyField)}
delete(record.Fields, keyCode)
@@ -114,6 +129,7 @@ func (bulk *BulkRequests) Request(app *kintone.App) (*DataResponseBulkPOST, inte
return resp1, nil
}
// Decode for BulkRequests
func (bulk *BulkRequests) Decode(b []byte) (*DataResponseBulkPOST, error) {
var rsp *DataResponseBulkPOST
err := json.Unmarshal(b, &rsp)
View
@@ -19,19 +19,19 @@ func newAppTest(id uint64) *kintone.App {
func TestRequest(t *testing.T) {
bulkReq := &BulkRequests{}
app := newAppTest(13)
app := newAppTest(16)
bulkReq.Requests = make([]*BulkRequestItem, 0)
/// INSERT
records := make([]*kintone.Record, 0)
record1 := kintone.NewRecord(map[string]interface{}{
"_1": kintone.SingleLineTextField("test 11!"),
"_2": kintone.SingleLineTextField("test 21!"),
"Text": kintone.SingleLineTextField("test 11!"),
"_2": kintone.SingleLineTextField("test 21!"),
})
records = append(records, record1)
record2 := kintone.NewRecord(map[string]interface{}{
"_1": kintone.SingleLineTextField("test 22!"),
"_2": kintone.SingleLineTextField("test 22!"),
"Text": kintone.SingleLineTextField("test 22!"),
"_2": kintone.SingleLineTextField("test 22!"),
})
records = append(records, record2)
dataPOST := &DataRequestRecordsPOST{app.AppId, records}
@@ -40,18 +40,18 @@ func TestRequest(t *testing.T) {
bulkReq.Requests = append(bulkReq.Requests, postRecords)
/// UPDATE
recordsUpdate := make([]*DataRequestRecordPUT, 0)
recordsUpdate := make([]interface{}, 0)
recordsUpdate1 := kintone.NewRecordWithId(4902, map[string]interface{}{
"_1": kintone.SingleLineTextField("test NNN!"),
"_2": kintone.SingleLineTextField("test MMM!"),
"Text": kintone.SingleLineTextField("test NNN!"),
"_2": kintone.SingleLineTextField("test MMM!"),
})
fmt.Println(recordsUpdate1)
recordsUpdate = append(recordsUpdate, &DataRequestRecordPUT{ID: recordsUpdate1.Id(),
Record: recordsUpdate1})
recordsUpdate2 := kintone.NewRecordWithId(4903, map[string]interface{}{
"_1": kintone.SingleLineTextField("test 123!"),
"_2": kintone.SingleLineTextField("test 234!"),
"Text": kintone.SingleLineTextField("test 123!"),
"_2": kintone.SingleLineTextField("test 234!"),
})
recordsUpdate = append(recordsUpdate, &DataRequestRecordPUT{
ID: recordsUpdate2.Id(), Record: recordsUpdate2})
View
@@ -5,25 +5,24 @@ import (
"fmt"
"io"
"os"
"path"
"regexp"
"strings"
"strconv"
"strings"
"time"
"path"
"errors"
"github.com/kintone/go-kintone"
"golang.org/x/text/transform"
)
type SubRecord struct {
Id uint64
Fields map[string]interface{}
Id uint64
Fields map[string]interface{}
}
func getReader(reader io.Reader) io.Reader {
encoding := getEncoding()
if (encoding == nil) {
if encoding == nil {
return reader
}
return transform.NewReader(reader, encoding.NewDecoder())
@@ -63,7 +62,6 @@ func addSubField(app *kintone.App, column *Column, col string, table *SubRecord)
}
func readCsv(app *kintone.App, _reader io.Reader) error {
reader := csv.NewReader(getReader(_reader))
head := true
@@ -278,27 +276,27 @@ func uploadFile(app *kintone.App, filePath string) (string, error) {
fileinfo, err := fi.Stat()
if err != nil {
return "", err
}
if err != nil {
return "", err
}
if fileinfo.Size() > 10 * 1024 * 1024 {
return "", errors.New(fmt.Sprintf("%s file must be less than 10 MB.", filePath))
if fileinfo.Size() > 10*1024*1024 {
return "", fmt.Errorf("%s file must be less than 10 MB", filePath)
}
fileKey, err := app.Upload(path.Base(filePath), "application/octet-stream", fi)
return fileKey, err
}
func insert(app *kintone.App, recs []*kintone.Record) error {
func insert(app *kintone.App, recs []*kintone.Record) error {
var err error
_, err = app.AddRecords(recs)
return err
}
func update(app *kintone.App, recs []*kintone.Record, keyField string) error {
func update(app *kintone.App, recs []*kintone.Record, keyField string) error {
var err error
if keyField != "" {
err = app.UpdateRecordsByKey(recs, true, keyField)
View
@@ -12,26 +12,13 @@ import (
"github.com/kintone/go-kintone"
)
const (
// ConstBulkRequestLimitRecordOption set option: record per bulkRequest
ConstBulkRequestLimitRecordOption = 100
// ConstBulkRequestLimitRequest kintone limited: The request count per bulkRequest
ConstBulkRequestLimitRequest = 20
// ConstRecordsLimitPerRequest kintone limited: The records count per request
ConstRecordsLimitPerRequest = 100
)
func importFromCSV(app *kintone.App, _reader io.Reader) error {
reader := csv.NewReader(getReader(_reader))
head := true
// recordsInsert := make([]*kintone.Record, 0, IMPORT_ROW_LIMIT)
// recordsUpdate := make([]*kintone.Record, 0, IMPORT_ROW_LIMIT)
var columns Columns
var lastRowImport uint64
lastRowImport = config.line
bulkRequests := &BulkRequests{}
View
53 main.go
@@ -14,6 +14,7 @@ import (
"golang.org/x/text/encoding/unicode"
)
// Configure cli configuration
type Configure struct {
login string
password string
@@ -38,7 +39,10 @@ type Configure struct {
var config Configure
// IMPORT_ROW_LIMIT Limit of kintone app (POST/PUT)
const IMPORT_ROW_LIMIT = 100
// IMPORT_ROW_LIMIT Limit of kintone app (GET)
const EXPORT_ROW_LIMIT = 500
type Column struct {
@@ -212,25 +216,54 @@ func main() {
}
var err error
if config.filePath == "" {
if config.isImport {
err = importFromCSV(app, os.Stdin)
} else {
// Old logic without force import/export
if config.isImport == false && config.isExport == false {
if config.filePath == "" {
if config.format == "json" {
err = writeJson(app, os.Stdout)
} else {
err = writeCsv(app, os.Stdout)
}
} else {
importDataFromFile(app)
}
} else {
var file *os.File
file, err = os.Open(config.filePath)
if err == nil {
defer file.Close()
err = importFromCSV(app, file)
}
// Filter flag: the first flag have priority
if config.isImport && config.isExport {
log.Fatal("The -f option is not supported with the --export option.")
}
if config.isImport {
if config.filePath == "" {
err = importFromCSV(app, os.Stdin)
} else {
importDataFromFile(app)
}
}
if config.isExport {
if config.filePath != "" {
log.Fatal("The options --import and --export cannot be specified together!")
}
if config.format == "json" {
err = writeJson(app, os.Stdout)
} else {
err = writeCsv(app, os.Stdout)
}
}
if err != nil {
log.Fatal(err)
}
}
func importDataFromFile(app *kintone.App) error {
var file *os.File
var err error
file, err = os.Open(config.filePath)
if err == nil {
defer file.Close()
err = importFromCSV(app, file)
}
return err
}

0 comments on commit 2dd0f84

Please sign in to comment.