/
staging.go
87 lines (74 loc) · 2.29 KB
/
staging.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package analytics
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/apigee/apigee-remote-service-golib/log"
"github.com/hashicorp/go-multierror"
)
func (m *manager) stageFile(tenant, tempFile string, numRecs int) {
stageDir := m.getStagingDir(tenant)
stagedFile := filepath.Join(stageDir, filepath.Base(tempFile))
if err := os.Rename(tempFile, stagedFile); err != nil {
log.Errorf("can't rename file: %s", err)
return
}
prometheusRecordsByFile.WithLabelValues(stagedFile).Set(float64(numRecs))
m.upload(tenant, stagedFile, numRecs)
log.Debugf("staged file: %s", stagedFile)
}
func (m *manager) getFilesInStaging() ([]string, error) {
tenantDirs, err := ioutil.ReadDir(m.stagingDir)
if err != nil {
return nil, fmt.Errorf("ReadDir(%s): %s", m.tempDir, err)
}
var errs error
var filePaths []string
for _, tenantDir := range tenantDirs {
tenantDirPath := filepath.Join(m.stagingDir, tenantDir.Name())
stagedFiles, err := ioutil.ReadDir(tenantDirPath)
if err != nil {
errs = multierror.Append(errs, fmt.Errorf("ls %s: %s", tenantDirPath, err))
continue
}
for _, stagedFile := range stagedFiles {
filePaths = append(filePaths, filepath.Join(tenantDirPath, stagedFile.Name()))
}
}
return filePaths, errs
}
func (m *manager) stageAllBucketsWait() {
wait := &sync.WaitGroup{}
m.stageAllBuckets(wait)
wait.Wait()
}
func (m *manager) stageAllBuckets(wait *sync.WaitGroup) {
m.bucketsLock.Lock()
buckets := m.buckets
m.buckets = map[string]*bucket{}
m.bucketsLock.Unlock()
for tenant, bucket := range buckets {
m.stageBucket(tenant, bucket, wait)
}
}
func (m *manager) stageBucket(tenant string, b *bucket, wait *sync.WaitGroup) {
if wait != nil {
wait.Add(1)
}
b.close(wait)
}