-
Notifications
You must be signed in to change notification settings - Fork 63
/
data_prep.yaml
62 lines (57 loc) · 2.04 KB
/
data_prep.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
{{ $config := .Values.dataPrepConfig }}
apiVersion: kubeflow.org/v2beta1
kind: MPIJob
metadata:
name: nlp-data-prep
labels:
app: nlp-data-prep
spec:
mpiReplicaSpecs:
Launcher:
replicas: 1
template:
spec:
containers:
- name: nlp-data-prep
image: {{ .Values.image.trainingImage }}
command: ["bash", "-c"]
args:
- '{{- range tuple "download" "extract" "preprocess" }} /opt/amazon/openmpi/bin/mpirun --allow-run-as-root -np {{ $config.totalProcesses }} -npernode {{ $config.procsPerNode }} -bind-to none -map-by slot --oversubscribe -x PYTHONPATH -mca pml ob1 -mca btl ^openib python3 /opt/NeMo-Megatron-Launcher/launcher_scripts/nemo_launcher/collections/dataprep_scripts/pile_dataprep/{{ . }}.py --config-path=/config --config-name=config.yaml && {{- end}} echo Data preparation complete'
imagePullPolicy: Always
imagePullSecrets:
- name: {{ .Values.image.pullSecret }}
Worker:
replicas: {{ .Values.image.nodes }}
template:
spec:
containers:
- name: nlp-data-prep
image: {{ .Values.image.trainingImage }}
command: ["/usr/sbin/sshd"]
args:
- "-De"
volumeMounts:
- mountPath: /dev/shm
name: dshm
- mountPath: /config
name: data-prep-config
- mountPath: ${PATH_TO_FSX}
name: ${FSX_PV}
imagePullPolicy: Always
restartPolicy: Never
imagePullSecrets:
- name: {{ .Values.image.pullSecret }}
volumes:
- name: ${FSX_PV}
persistentVolumeClaim:
claimName: ${FSX_PVC}
- name: dshm
emptyDir:
medium: Memory
sizeLimit: {{ $config.shmSize }}
- configMap:
name: data-prep-config
name: data-prep-config
{{ if ne $config.dnsPolicy "nil" }}
dnsPolicy: {{ $config.dnsPolicy }}
{{ end }}