/
zipdata.py
161 lines (132 loc) · 4.71 KB
/
zipdata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
import os
import sys
from glob import glob
from shutil import rmtree
from zipfile import ZipFile
from ..parameters import ZIP_OPTIONS
from ..core.helpers import console, splitModRef
GH_BASE = os.path.expanduser(f'~/github')
DW_BASE = os.path.expanduser(f'~/Downloads')
TEMP = '_temp'
RELATIVE = 'tf'
HELP = '''
USAGE
text-fabric-zip --help
text-fabric-zip {org}/{repo}/{relative}
EFFECT
Zips text-fabric data from your local github repository into
a release file, ready to be attached to a github release.
Your repo must sit in ~/github/{org}/{repo}.
Your TF data is assumed to sit in the toplevel tf directory of your repo.
But if it is somewhere else, you can pass relative, e.g phrases/heads/tf
It is assumed that your tf directory contains subdirectories according to
the versions of the main datasource.
The actual .tf files are in those version directories.
Each of these version directories will be zipped into a separate file.
The resulting zip files end up in ~/Downloads/{org}-release/{repo}
and the are named {relative}-{version}.zip
(where the / in relative have been replaced by -)
'''
EXCLUDE = {'.DS_Store'}
def zipData(org, repo, relative=RELATIVE, tf=True, keep=False):
console(f'Create release data for {org}/{repo}/{relative}')
sourceBase = f'{GH_BASE}/{org}'
destBase = f'{DW_BASE}/{org}-release'
sourceDir = f'{sourceBase}/{repo}/{relative}'
destDir = f'{destBase}/{repo}'
dataFiles = {}
if not keep:
if os.path.exists(destDir):
rmtree(destDir)
os.makedirs(destDir, exist_ok=True)
relativeDest = relative.replace('/', '-')
if tf:
versionEntries = []
for versionEntry in glob(f'{sourceDir}/*'):
if os.path.isdir(versionEntry):
(versionDir, version) = os.path.split(versionEntry)
versionEntries.append((versionDir, version))
if versionEntries:
console(f'Found {len(versionEntries)} versions')
else:
versionEntries.append((sourceDir, ''))
console(f'Found unversioned features')
for (versionDir, version) in versionEntries:
if version == TEMP:
continue
versionRep = f'/{version}' if version else ''
versionRep2 = f'{version}/' if version else ''
versionRep3 = f'-{version}' if version else ''
for tfEntry in glob(f'{versionDir}{versionRep}/*'):
if not os.path.isfile(tfEntry):
continue
(tfDir, featureFile) = os.path.split(tfEntry)
if featureFile in EXCLUDE:
continue
if not featureFile.endswith('.tf'):
console(f'WARNING: non feature file "{versionRep2}{featureFile}"', error=True)
continue
dataFiles.setdefault(version, set()).add(featureFile)
console(f'zip files end up in {destDir}')
for (version, features) in sorted(dataFiles.items()):
item = f'{org}/{repo}'
versionRep = f'/{version}' if version else ''
versionRep3 = f'-{version}' if version else ''
target = f'{relativeDest}{versionRep3}.zip'
console(f'zipping {item:<25} {version:>4} with {len(features):>3} features ==> {target}')
with ZipFile(
f'{destDir}/{target}',
'w',
**ZIP_OPTIONS,
) as zipFile:
for featureFile in sorted(features):
zipFile.write(
f'{sourceDir}{versionRep}/{featureFile}',
arcname=featureFile,
)
else:
def collectFiles(base, path, results):
thisPath = f'{base}/{path}' if path else base
internalBase = f'{relative}/{path}' if path else relative
for entry in glob(f'{thisPath}/*'):
(entryDir, name) = os.path.split(entry)
if name in EXCLUDE:
continue
if os.path.isfile(entry):
results.append((f'{internalBase}/{name}', f'{base}/{path}/{name}'))
elif os.path.isdir(entry):
collectFiles(base, f'{path}/{name}', results)
results = []
collectFiles(sourceDir, '', results)
if not relativeDest:
relativeDest = '-'
console(f'zipping {org}/{repo}/{relative} with {len(results)} files')
console(f'zip file is {destDir}/{relativeDest}.zip')
with ZipFile(
f'{destDir}/{relativeDest}.zip',
'w',
**ZIP_OPTIONS,
) as zipFile:
for (internalPath, path) in sorted(results):
zipFile.write(
path,
arcname=internalPath,
)
def main(cargs=sys.argv):
if len(cargs) != 2 and any(arg in {'--help', '-help', '-h', '?', '-?'} for arg in cargs):
console(HELP)
return
moduleRef = cargs[1]
parts = splitModRef(moduleRef)
if not parts:
console(HELP)
return
(org, repo, relative, checkout) = parts
tf = (
relative.endswith('tf') or
'/tf/' in relative
)
print(tf)
zipData(org, repo, relative=relative, tf=tf)
if __name__ == "__main__":
main()