From e950b0327974b26329bad92dd75b968113ec5f44 Mon Sep 17 00:00:00 2001 From: Katie Horne Date: Fri, 19 Mar 2021 12:50:06 -0500 Subject: [PATCH 1/3] chore: add doc on increasing inotify-watch-limits --- guides/inotify-watch-limits.md | 140 +++++++++++++++++++++++++++++++++ manifest.json | 1 + 2 files changed, 141 insertions(+) create mode 100644 guides/inotify-watch-limits.md diff --git a/guides/inotify-watch-limits.md b/guides/inotify-watch-limits.md new file mode 100644 index 000000000..5e492f121 --- /dev/null +++ b/guides/inotify-watch-limits.md @@ -0,0 +1,140 @@ +--- +title: Inotify Watch Limit Increase +description: Learn how to increase the inotify watcher limit. +--- + +When running a webpack project, you may encounter an error similar to the +following: + +```text +Watchpack Error (watcher): Error: ENOSPC: System limit for number of file +watchers reached, watch '/some/path' +``` + +## Resolution + +Increase the number of file watchers. Because the setting quantifying the number +of file watchers isn't namespaced, you'll need to raise the maximum number at +the node level. + +One way to do this is to use a daemonset with a privileged container to set the +maximum number of file watchers (H/T: +[xinyanmsft](https://github.com/Azure/AKS/issues/772#issuecomment-477760184)): + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: more-fs-watchers + namespace: kube-system + labels: + app: more-fs-watchers +spec: + template: + metadata: + labels: + name: more-fs-watchers + spec: + hostNetwork: true + hostPID: true + hostIPC: true + initContainers: + - command: + - sh + - -c + - sysctl -w fs.inotify.max_user_watches=524288; + image: alpine:3.6 + imagePullPolicy: IfNotPresent + name: sysctl + resources: {} + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + containers: + - resources: + requests: + cpu: 0.01 + image: alpine:3.6 + name: sleepforever + command: ["tail"] + args: ["-f", "/dev/null"] + volumes: + - name: sys + hostPath: + path: /sys +``` + +For Kubernetes v1.18: + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: more-fs-watchers + namespace: kube-system + labels: + app: more-fs-watchers + k8s-app: more-fs-watchers +spec: + selector: + matchLabels: + k8s-app: more-fs-watchers + template: + metadata: + labels: + name: more-fs-watchers + k8s-app: more-fs-watchers + spec: + hostNetwork: true + hostPID: true + hostIPC: true + initContainers: + - command: + - sh + - -c + - sysctl -w fs.inotify.max_user_watches=524288; + image: alpine:3.6 + imagePullPolicy: IfNotPresent + name: sysctl + resources: {} + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + containers: + - resources: + requests: + cpu: 0.01 + image: alpine:3.6 + name: sleepforever + command: ["tail"] + args: ["-f", "/dev/null"] + volumes: + - name: sys + hostPath: + path: /sys +``` + +## Notes + +- Daemonsets without node selectors will persist on the cluster and will run on + *every* node. Every new node that you spin up will also have an associated + daemonset pod spun up to ensure that the sysctl command runs on every node. +- You can **delete** this by running: + + ```console + kubectl delete more-fs-watchers + ``` + + This command removes all related daemonset pods, and no further pods will be + spun up. + +## Helpful Resources + +- [Setting Sysctls for a + Pod](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#setting-sysctls-for-a-pod) +- [Increasing the Amount of `inotify` + Watchers](https://github.com/guard/listen/blob/master/README.md#increasing-the-amount-of-inotify-watchers) diff --git a/manifest.json b/manifest.json index b0737f1f8..b4bd36ba1 100644 --- a/manifest.json +++ b/manifest.json @@ -116,6 +116,7 @@ { "path": "./guides/custom-env.md" }, { "path": "./guides/gitconfig.md" }, { "path": "./guides/helm-charts.md" }, + { "path": "./guides/inotify-watch-limits.md" }, { "path": "./guides/macos-keybinding.md" }, { "path": "./guides/mobile-development.md" }, { "path": "./guides/nightly-releases.md" }, From 8e11fd1a863d0caa9dfa183f2fd244625b1cd1e7 Mon Sep 17 00:00:00 2001 From: Katie Horne Date: Fri, 19 Mar 2021 12:54:40 -0500 Subject: [PATCH 2/3] Add text --- guides/inotify-watch-limits.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/guides/inotify-watch-limits.md b/guides/inotify-watch-limits.md index 5e492f121..e9ee6ef01 100644 --- a/guides/inotify-watch-limits.md +++ b/guides/inotify-watch-limits.md @@ -11,13 +11,17 @@ Watchpack Error (watcher): Error: ENOSPC: System limit for number of file watchers reached, watch '/some/path' ``` +This results from a low number of [inotify +watches](https://confluence.jetbrains.com/display/IDEADEV/Inotify+Watches+Limit) +combined with high node usage, causing the log stream to fail. + ## Resolution Increase the number of file watchers. Because the setting quantifying the number of file watchers isn't namespaced, you'll need to raise the maximum number at the node level. -One way to do this is to use a daemonset with a privileged container to set the +One way to do this is to use a daemonset in the cluster with a privileged container to set the maximum number of file watchers (H/T: [xinyanmsft](https://github.com/Azure/AKS/issues/772#issuecomment-477760184)): From 7becd9aa49d5c0902e4b50f0d517ece2a7b32117 Mon Sep 17 00:00:00 2001 From: Katie Horne Date: Fri, 19 Mar 2021 15:06:02 -0500 Subject: [PATCH 3/3] Remove extra example; lint --- guides/inotify-watch-limits.md | 52 ++-------------------------------- 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/guides/inotify-watch-limits.md b/guides/inotify-watch-limits.md index e9ee6ef01..bd408dfd2 100644 --- a/guides/inotify-watch-limits.md +++ b/guides/inotify-watch-limits.md @@ -21,56 +21,8 @@ Increase the number of file watchers. Because the setting quantifying the number of file watchers isn't namespaced, you'll need to raise the maximum number at the node level. -One way to do this is to use a daemonset in the cluster with a privileged container to set the -maximum number of file watchers (H/T: -[xinyanmsft](https://github.com/Azure/AKS/issues/772#issuecomment-477760184)): - -```yaml -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: more-fs-watchers - namespace: kube-system - labels: - app: more-fs-watchers -spec: - template: - metadata: - labels: - name: more-fs-watchers - spec: - hostNetwork: true - hostPID: true - hostIPC: true - initContainers: - - command: - - sh - - -c - - sysctl -w fs.inotify.max_user_watches=524288; - image: alpine:3.6 - imagePullPolicy: IfNotPresent - name: sysctl - resources: {} - securityContext: - privileged: true - volumeMounts: - - name: sys - mountPath: /sys - containers: - - resources: - requests: - cpu: 0.01 - image: alpine:3.6 - name: sleepforever - command: ["tail"] - args: ["-f", "/dev/null"] - volumes: - - name: sys - hostPath: - path: /sys -``` - -For Kubernetes v1.18: +One way to do this is to use a daemonset in the cluster with a privileged +container to set the maximum number of file watchers: ```yaml apiVersion: apps/v1